Skip to main content

dsl_kit_parse/
schema_gen.rs

1//! Schema-driven grammar generation (item N).
2//!
3//! The kit's identity axis is "write the AST in Rust, the toolchain
4//! supplies the rest". [`grammar_from_schema`] converts the type-level
5//! shape a `#[derive(DslSchema)]` already extracts into a working
6//! [`Grammar`] for a canonical text syntax — the DSL author writes two
7//! derives and gets an external-format parser for free, instead of
8//! hand-authoring a grammar (in Rust or, worse, in a Bison-style JSON
9//! blob nobody wants to write).
10//!
11//! # Canonical syntax
12//!
13//! One form per variant: the variant name as a keyword, followed by a
14//! parenthesised, comma-separated argument list. Arguments appear in
15//! schema order — payload fields first, then child slots — each spelled
16//! `name: value`:
17//!
18//! ```text
19//! Add(lhs: Lit(value: 1), rhs: Neg(body: none))
20//! List(items: [Lit(value: 1), Lit(value: 2)])
21//! Name(text: "hello world")
22//! Unit()
23//! ```
24//!
25//! - **`One` child** — `name: <node>`.
26//! - **`Optional` child** — `name: <node>` or `name: none`. (If a
27//!   variant is literally called `none`, PEG ordered choice still
28//!   parses `name: none(...)` as that variant; bare `none` means
29//!   absent.)
30//! - **`Many` child** — `name: [<node>, <node>, ...]`, empty list
31//!   allowed.
32//! - **`Map` child** — `name: { <key>: <node>, <key>: <node> }`, empty
33//!   map allowed. Braces (not the `Many` brackets) mark the slot as
34//!   keyed, matching the JSON bridge's object spelling. A key is
35//!   either a `%str` literal or a bare `%ident`, so `"a b": …` and
36//!   `path: …` are both writable. Entries may be written in any
37//!   order — the parser sorts them into the canonical
38//!   ascending-by-key form the `ParseTree` requires.
39//! - **Payload fields** — by Rust type source text:
40//!   `String` → a `%str` literal; the integer types → `%int`;
41//!   `bool` → `true` / `false`. Any other payload type fails
42//!   generation with [`codes::UNSUPPORTED_FIELD`] — loudly, per
43//!   variant, rather than silently dropping the variant from the
44//!   grammar — unless a [`SyntaxOverrides`] entry supplies its value
45//!   production.
46//!
47//! # Syntax overrides
48//!
49//! [`grammar_from_schema_with`] accepts a [`SyntaxOverrides`] carrying
50//! value productions for payload types the built-in mapping rejects
51//! (e.g. `Option<JoinPolicy>`), keyed by type source text or pinned to
52//! one `(variant, field)` site. The override supplies only the value
53//! [`Peg`] — the surrounding `name: value` argument shape stays
54//! canonical — and its matched text becomes the field's production, so
55//! the override author owns making that text consumable downstream
56//! (typically `FromStr` on the field type, via
57//! [`build_field`](crate::build_field)).
58//!
59//! # Guarantees
60//!
61//! The generated grammar is left-recursion-free by construction (every
62//! variant rule starts with its keyword token) and contains no nullable
63//! repeats; `GrammarCheck`'s `check_left_recursion` /
64//! `check_nullable_repeat` / `check_schema_consistency` all pass
65//! against it — the tests pin this. Trees produced by parsing satisfy
66//! [`check_conformance`](crate::check_conformance) against the source
67//! schema.
68
69use std::collections::HashMap;
70
71use crate::grammar_check;
72use crate::peg::{
73    Grammar, Peg, choice, field, keyed_entry, node, repeat, rule, rule_ref, seq, token,
74};
75use crate::{BuildError, Diagnostic};
76use dsl_kit_core::IdGen;
77use dsl_kit_schema::{
78    ChildSchema, ChildValueShape, FieldSchema, Multiplicity, NodeSchema, VariantSchema,
79};
80
81/// Diagnostic codes emitted by grammar generation.
82pub mod codes {
83    /// A payload field's Rust type has no canonical-syntax mapping.
84    pub const UNSUPPORTED_FIELD: &str = "dsl_kit::schema_gen::unsupported_field";
85    /// The schema declares no variants — there is nothing to parse.
86    pub const EMPTY_SCHEMA: &str = "dsl_kit::schema_gen::empty_schema";
87    /// A child slot's multiplicity is a variant this build of
88    /// `dsl-kit-parse` does not recognise (added to the schema crate's
89    /// `#[non_exhaustive]` enum ahead of parse-side support). Emitted
90    /// by the pre-flight check in
91    /// [`grammar_from_schema_with`] so `child_arg_peg` never sees an
92    /// unknown variant.
93    pub const UNKNOWN_MULTIPLICITY: &str = "dsl_kit::schema_gen::unknown_multiplicity";
94    /// A keyed child slot declares a value shape the canonical text
95    /// syntax has not yet grown a production for. Emitted by the
96    /// pre-flight check so a scalar-valued keyed slot fails loudly
97    /// at grammar-generation time rather than silently producing a
98    /// grammar that expects each entry's value to be a full AST
99    /// node — the shape mismatch would otherwise surface as an
100    /// opaque parse failure on the DSL author's first document.
101    pub const UNSUPPORTED_MAP_VALUE_SHAPE: &str =
102        "dsl_kit::schema_gen::unsupported_map_value_shape";
103}
104
105/// Name of the generated start rule.
106pub const START_RULE: &str = "node";
107
108/// A deferred value-production constructor: overrides are registered
109/// before the [`IdGen`] doing the generation exists, so each entry is a
110/// factory invoked with the generator's `ids` at build time.
111type PegFactory = Box<dyn Fn(&IdGen) -> Peg>;
112
113/// Value-production overrides for payload fields whose Rust type has no
114/// built-in canonical-syntax mapping (see the module docs).
115///
116/// Two granularities, resolved most-specific-first:
117///
118/// 1. **Per field** ([`Self::for_field`]) — pinned to one
119///    `(variant, field)` site.
120/// 2. **Per type** ([`Self::for_type`]) — keyed by the field's Rust
121///    type source text; whitespace is ignored, so
122///    `"Option<JoinPolicy>"` matches the derive-extracted
123///    `"Option < JoinPolicy >"` spelling.
124///
125/// A field with neither entry falls back to the built-in mapping, and
126/// failing that still fails generation loudly with
127/// [`codes::UNSUPPORTED_FIELD`].
128///
129/// The factory builds only the **value** production; the enclosing
130/// `name ':' <value>` argument shape (and the `Field` sink binding the
131/// matched text to the field name) stays canonical, which keeps the
132/// generated grammar's `GrammarCheck` guarantees intact.
133#[derive(Default)]
134pub struct SyntaxOverrides {
135    by_type: HashMap<String, PegFactory>,
136    by_field: HashMap<(String, String), PegFactory>,
137}
138
139impl SyntaxOverrides {
140    /// An empty override set (equivalent to the built-in mapping only).
141    pub fn new() -> Self {
142        Self::default()
143    }
144
145    /// Registers a value production for every payload field of Rust
146    /// type `ty` (source-text comparison, whitespace-insensitive).
147    pub fn for_type(
148        mut self,
149        ty: impl AsRef<str>,
150        value: impl Fn(&IdGen) -> Peg + 'static,
151    ) -> Self {
152        self.by_type.insert(strip_ws(ty.as_ref()), Box::new(value));
153        self
154    }
155
156    /// Registers a value production for the single field `field` of
157    /// variant `variant`, winning over any [`Self::for_type`] entry.
158    pub fn for_field(
159        mut self,
160        variant: impl Into<String>,
161        field: impl Into<String>,
162        value: impl Fn(&IdGen) -> Peg + 'static,
163    ) -> Self {
164        self.by_field
165            .insert((variant.into(), field.into()), Box::new(value));
166        self
167    }
168
169    /// Most-specific matching factory for `field` of `variant`, if any.
170    fn resolve(&self, variant: &str, field: &FieldSchema) -> Option<&PegFactory> {
171        self.by_field
172            .get(&(variant.to_string(), field.name.clone()))
173            .or_else(|| self.by_type.get(&strip_ws(&field.ty)))
174    }
175}
176
177/// Whitespace-insensitive type key: `Option < JoinPolicy >` (the
178/// token-stream spelling `#[derive(DslSchema)]` extracts) and
179/// `Option<JoinPolicy>` (what an author writes) must compare equal.
180fn strip_ws(ty: &str) -> String {
181    ty.chars().filter(|c| !c.is_whitespace()).collect()
182}
183
184impl Grammar {
185    /// Generates the canonical-syntax grammar for `schema`. See
186    /// [`grammar_from_schema`].
187    pub fn from_schema(schema: &NodeSchema, ids: &IdGen) -> Result<Grammar, BuildError> {
188        grammar_from_schema(schema, ids)
189    }
190}
191
192/// Generates a [`Grammar`] for the canonical named-argument syntax of
193/// `schema` (see the module docs for the syntax), built-in field
194/// mapping only.
195///
196/// Fails with one [`codes::UNSUPPORTED_FIELD`] diagnostic per
197/// unmappable payload field (collected across all variants, so the
198/// author sees the full list at once). To map such fields instead of
199/// failing, use [`grammar_from_schema_with`].
200pub fn grammar_from_schema(schema: &NodeSchema, ids: &IdGen) -> Result<Grammar, BuildError> {
201    grammar_from_schema_with(schema, ids, &SyntaxOverrides::default())
202}
203
204/// [`grammar_from_schema`] with [`SyntaxOverrides`] consulted before
205/// the built-in field mapping (per-field entries win over per-type
206/// entries win over built-ins).
207pub fn grammar_from_schema_with(
208    schema: &NodeSchema,
209    ids: &IdGen,
210    overrides: &SyntaxOverrides,
211) -> Result<Grammar, BuildError> {
212    if schema.variants.is_empty() {
213        return Err(BuildError::single(Diagnostic::error(
214            codes::EMPTY_SCHEMA,
215            format!("schema `{}` declares no variants", schema.name),
216        )));
217    }
218
219    // Pre-flight: collect every diagnostic that must block rule
220    // generation, then return in one shot. A single pass means an
221    // author with both a bad payload type and a `Multiplicity::Map`
222    // slot sees both errors on the first build instead of paying two
223    // edit-compile cycles for one report.
224    //
225    // Grouping:
226    //   - `UNSUPPORTED_FIELD` for payload types with no canonical-syntax
227    //     mapping (and no `SyntaxOverrides` entry).
228    //   - `UNKNOWN_MULTIPLICITY` for child slots whose multiplicity
229    //     this build does not know, which is what keeps
230    //     `child_arg_peg`'s catch-all arm from ever running.
231    let mut pre_flight = Vec::new();
232    for v in &schema.variants {
233        for f in &v.fields {
234            if resolved_field_value_peg(&v.name, f, ids, overrides).is_none() {
235                pre_flight.push(Diagnostic::error(
236                    codes::UNSUPPORTED_FIELD,
237                    format!(
238                        "variant `{}` field `{}`: type `{}` has no canonical-syntax \
239                         mapping (supported: String, bool, the integer types; or \
240                         register a `SyntaxOverrides` value production)",
241                        v.name, f.name, f.ty
242                    ),
243                ));
244            }
245        }
246        for c in &v.children {
247            match c.multiplicity {
248                // Every shape `child_arg_peg` knows how to emit.
249                Multiplicity::One | Multiplicity::Optional | Multiplicity::Many => {}
250                // Keyed slots come in two value shapes at derive
251                // level: recursive (`BTreeMap<String, Self>` /
252                // `BTreeMap<String, Box<Self>>`) is supported by the
253                // canonical text syntax; scalar
254                // (`ChildValueShape::Scalar { .. }`, Shape 1 of the
255                // tracking issue) has not grown a text production
256                // yet — `child_arg_peg` would emit
257                // `key: <node>` for every entry, producing an
258                // unusable grammar. Fail loudly instead so the DSL
259                // author is directed to the JSON front-end.
260                Multiplicity::Map => {
261                    if let ChildValueShape::Scalar { ty } = &c.value_shape {
262                        pre_flight.push(Diagnostic::error(
263                            codes::UNSUPPORTED_MAP_VALUE_SHAPE,
264                            format!(
265                                "variant `{}` child slot `{}`: keyed scalar values \
266                                 (`BTreeMap<String, {}>`) are recognised by the derive and \
267                                 the JSON front-end, but the canonical text syntax has no \
268                                 production for them yet — use the JSON front-end for \
269                                 scalar-map DSLs, or open the tracking issue if the text \
270                                 syntax is needed",
271                                v.name, c.name, ty
272                            ),
273                        ));
274                    }
275                }
276                // `#[non_exhaustive]` catch-all: a future Multiplicity
277                // variant added to dsl-kit-schema before this crate is
278                // updated must also be rejected here — that's what
279                // keeps `child_arg_peg`'s `_ => unreachable!()` arm
280                // honest.
281                _ => {
282                    pre_flight.push(Diagnostic::error(
283                        codes::UNKNOWN_MULTIPLICITY,
284                        format!(
285                            "variant `{}` child slot `{}`: unrecognised Multiplicity \
286                             variant (upgrade dsl-kit-parse to a version that knows about it)",
287                            v.name, c.name
288                        ),
289                    ));
290                }
291            }
292        }
293    }
294    if !pre_flight.is_empty() {
295        return Err(BuildError::new(pre_flight));
296    }
297
298    let mut rules = Vec::with_capacity(schema.variants.len() + 1);
299    rules.push(rule(
300        ids,
301        START_RULE,
302        choice(
303            ids,
304            schema
305                .variants
306                .iter()
307                .map(|v| rule_ref(ids, v.name.clone()))
308                .collect(),
309        ),
310    ));
311    for v in &schema.variants {
312        rules.push(variant_rule(v, ids, overrides));
313    }
314    Ok(Grammar::new(rules, START_RULE))
315}
316
317/// Builds the rule for one variant. Two shapes:
318///
319/// - **Strict-order** (default, variant has no optional fields):
320///   `%kw:V "(" arg ("," arg)* ")"` with arguments emitted in schema
321///   order (fields, then children). Preserves the pre-0.3 canonical
322///   spelling for variants that never omit.
323/// - **Free-order** (variant has ≥ 1 optional field): `%kw:V "(" (arg
324///   ("," arg)*)? ")"` where each arg is `arg_1 | arg_2 | ...` and
325///   every alternative carries its own `%kw:name` prefix. Args are
326///   distinguishable by that name-keyword, so the DSL author may
327///   omit any subset of optional arguments (or even all of them),
328///   and `check_conformance` remains the authority on required /
329///   duplicate / unknown-slot diagnostics.
330fn variant_rule(v: &VariantSchema, ids: &IdGen, overrides: &SyntaxOverrides) -> Peg {
331    let has_optional = v.fields.iter().any(|f| f.optional);
332    let mut args: Vec<Peg> = Vec::new();
333    for f in &v.fields {
334        let value = resolved_field_value_peg(&v.name, f, ids, overrides)
335            .expect("unsupported field types were rejected before rule generation");
336        args.push(seq(
337            ids,
338            vec![
339                token(ids, format!("%kw:{}", f.name)),
340                token(ids, ":"),
341                field(ids, f.name.clone(), value),
342            ],
343        ));
344    }
345    for c in &v.children {
346        args.push(child_arg_peg(c, ids));
347    }
348
349    let mut items = vec![token(ids, format!("%kw:{}", v.name)), token(ids, "(")];
350    if has_optional && !args.is_empty() {
351        // Free-order: any of the argument shapes, comma-separated,
352        // whole list optional. Each argument-slot is a Field capture
353        // in its own right (built above), and the name-keyword prefix
354        // keeps the alternation unambiguous at the byte level.
355        let arg_choice = choice(ids, args);
356        // The tail repeats `("," arg_choice)`; we rebuild arg_choice
357        // per position so each Choice node gets a fresh id (Peg nodes
358        // are id-unique in a grammar).
359        let tail_alt = |ids: &IdGen| -> Peg {
360            let mut alts: Vec<Peg> = Vec::new();
361            for f in &v.fields {
362                let value = resolved_field_value_peg(&v.name, f, ids, overrides)
363                    .expect("unsupported field types were rejected before rule generation");
364                alts.push(seq(
365                    ids,
366                    vec![
367                        token(ids, format!("%kw:{}", f.name)),
368                        token(ids, ":"),
369                        field(ids, f.name.clone(), value),
370                    ],
371                ));
372            }
373            for c in &v.children {
374                alts.push(child_arg_peg(c, ids));
375            }
376            choice(ids, alts)
377        };
378        let tail = repeat(ids, seq(ids, vec![token(ids, ","), tail_alt(ids)]), 0, None);
379        let full = seq(ids, vec![arg_choice, tail]);
380        items.push(repeat(ids, full, 0, Some(1)));
381    } else {
382        for (i, arg) in args.into_iter().enumerate() {
383            if i > 0 {
384                items.push(token(ids, ","));
385            }
386            items.push(arg);
387        }
388    }
389    items.push(token(ids, ")"));
390
391    rule(
392        ids,
393        v.name.clone(),
394        node(ids, v.name.clone(), seq(ids, items)),
395    )
396}
397
398/// Value production for a payload field: the most specific
399/// [`SyntaxOverrides`] entry if registered, else the built-in mapping.
400fn resolved_field_value_peg(
401    variant: &str,
402    f: &FieldSchema,
403    ids: &IdGen,
404    overrides: &SyntaxOverrides,
405) -> Option<Peg> {
406    match overrides.resolve(variant, f) {
407        Some(factory) => Some(factory(ids)),
408        None => field_value_peg(f, ids),
409    }
410}
411
412/// Built-in value production for a payload field, by Rust type source
413/// text. `None` when the type has no canonical-syntax mapping.
414///
415/// Recognised (whitespace-insensitive on the type-source spelling):
416///
417/// - `String` → `%str`
418/// - the integer types → `%int`
419/// - `bool` → `true` / `false`
420/// - `Option<String>` → `none` | `%str`
421/// - `Vec<String>` → `[ %str ("," %str)* ]`, empty list allowed
422///
423/// The last two entries realise Layer 2 of the "built-in optional
424/// payload fields" contract — downstream DSLs no longer need to
425/// register a [`SyntaxOverrides`] entry (nor a paired
426/// `#[dsl_build(with = ...)]` converter) for these shapes.
427fn field_value_peg(f: &FieldSchema, ids: &IdGen) -> Option<Peg> {
428    const INT_TYPES: &[&str] = &[
429        "i8", "i16", "i32", "i64", "i128", "u8", "u16", "u32", "u64", "u128", "usize", "isize",
430    ];
431    let ty = strip_ws(&f.ty);
432    if ty == "String" {
433        Some(token(ids, "%str"))
434    } else if INT_TYPES.contains(&ty.as_str()) {
435        Some(token(ids, "%int"))
436    } else if ty == "bool" {
437        Some(choice(
438            ids,
439            vec![token(ids, "%kw:true"), token(ids, "%kw:false")],
440        ))
441    } else if ty == "Option<String>" {
442        // Canonical spelling of an absent Option-payload; mirrors the
443        // `Multiplicity::Optional` child idiom.
444        Some(choice(
445            ids,
446            vec![token(ids, "%kw:none"), token(ids, "%str")],
447        ))
448    } else if ty == "Vec<String>" {
449        // `[ %str_raw ("," %str_raw)* ]` with empty list allowed.
450        // `%str_raw` (peg-level primitive) contributes the raw
451        // source slice for each element — quotes and escapes intact —
452        // so the joined field text is a JSON-compatible array literal
453        // that `build_field_vec` can hand straight to
454        // `serde_json::from_str`.
455        let elems = seq(
456            ids,
457            vec![
458                token(ids, "%str_raw"),
459                repeat(
460                    ids,
461                    seq(ids, vec![token(ids, ","), token(ids, "%str_raw")]),
462                    0,
463                    None,
464                ),
465            ],
466        );
467        Some(seq(
468            ids,
469            vec![
470                token(ids, "["),
471                repeat(ids, elems, 0, Some(1)),
472                token(ids, "]"),
473            ],
474        ))
475    } else {
476        None
477    }
478}
479
480/// Argument production for one child slot, by multiplicity.
481fn child_arg_peg(c: &ChildSchema, ids: &IdGen) -> Peg {
482    let name_kw = token(ids, format!("%kw:{}", c.name));
483    let colon = token(ids, ":");
484    match c.multiplicity {
485        Multiplicity::One => seq(
486            ids,
487            vec![
488                name_kw,
489                colon,
490                field(ids, c.name.clone(), rule_ref(ids, START_RULE)),
491            ],
492        ),
493        // `name: <node>` or `name: none`. The Field arm is tried first,
494        // so a variant literally named `none` still wins when followed
495        // by `(`. The bare-`none` arm sits *outside* any Field: its
496        // matched text lands on the enclosing Node sink where it is
497        // dropped as syntactic noise, leaving the slot absent.
498        Multiplicity::Optional => seq(
499            ids,
500            vec![
501                name_kw,
502                colon,
503                choice(
504                    ids,
505                    vec![
506                        field(ids, c.name.clone(), rule_ref(ids, START_RULE)),
507                        token(ids, "%kw:none"),
508                    ],
509                ),
510            ],
511        ),
512        // `name: [ <node> ("," <node>)* ]`, empty list allowed. The
513        // list commas live inside the Field body; the mixed-field
514        // "trees win" fallback drops them as noise.
515        Multiplicity::Many => {
516            let elems = seq(
517                ids,
518                vec![
519                    rule_ref(ids, START_RULE),
520                    repeat(
521                        ids,
522                        seq(ids, vec![token(ids, ","), rule_ref(ids, START_RULE)]),
523                        0,
524                        None,
525                    ),
526                ],
527            );
528            seq(
529                ids,
530                vec![
531                    name_kw,
532                    colon,
533                    token(ids, "["),
534                    field(ids, c.name.clone(), repeat(ids, elems, 0, Some(1))),
535                    token(ids, "]"),
536                ],
537            )
538        }
539        // `name: { <key>: <node> ("," <key>: <node>)* }`, empty map
540        // allowed. Braces rather than the `Many` idiom's brackets so
541        // the two shapes are distinguishable at a glance, and matching
542        // the JSON bridge's object spelling so a DSL reads the same
543        // way in both front-ends.
544        //
545        // The key token is `%str | %ident`: keys are arbitrary strings
546        // at the `ParseTree` level, so a quoted form has to exist, but
547        // requiring quotes around every `PATH`-style key would be
548        // noise. `%str` is tried first — a quoted key is unambiguous,
549        // while `%ident` cannot start with `"` — and its production is
550        // the decoded content, so `"a b"` and `a` both arrive as plain
551        // key text.
552        Multiplicity::Map => {
553            //
554            // The `:` separator rides on the *value* half, not the
555            // key half: every token matched inside the key production
556            // contributes to the key text, so a `:` there would land
557            // in the key itself. On the value side it is dropped as
558            // syntactic noise, the same way the `Many` list's commas
559            // are.
560            let entry = |ids: &IdGen| -> Peg {
561                keyed_entry(
562                    ids,
563                    c.name.clone(),
564                    choice(ids, vec![token(ids, "%str"), token(ids, "%ident")]),
565                    seq(ids, vec![token(ids, ":"), rule_ref(ids, START_RULE)]),
566                )
567            };
568            let entries = seq(
569                ids,
570                vec![
571                    entry(ids),
572                    repeat(ids, seq(ids, vec![token(ids, ","), entry(ids)]), 0, None),
573                ],
574            );
575            seq(
576                ids,
577                vec![
578                    name_kw,
579                    colon,
580                    token(ids, "{"),
581                    repeat(ids, entries, 0, Some(1)),
582                    token(ids, "}"),
583                ],
584            )
585        }
586        // `#[non_exhaustive]` catch-all: a future Multiplicity variant
587        // added to dsl-kit-schema before this crate is updated would
588        // land here. The pre-flight check in
589        // `grammar_from_schema_with` should be extended in lockstep to
590        // reject unknown variants; if that extension is missed the
591        // panic here surfaces the omission loudly instead of emitting
592        // a silently-wrong grammar rule.
593        _ => unreachable!(
594            "child_arg_peg encountered an unrecognised Multiplicity variant on slot \
595             `{}` — extend the pre-flight check in grammar_from_schema_with to reject it",
596            c.name
597        ),
598    }
599}
600
601/// Convenience: generate and immediately assert the grammar is clean
602/// under `GrammarCheck` (left recursion, nullable repeats, schema
603/// consistency). Intended for tests and one-shot setup paths; returns
604/// the same grammar as [`grammar_from_schema`].
605pub fn checked_grammar_from_schema(
606    schema: &NodeSchema,
607    ids: &IdGen,
608) -> Result<Grammar, BuildError> {
609    checked_grammar_from_schema_with(schema, ids, &SyntaxOverrides::default())
610}
611
612/// [`checked_grammar_from_schema`] with [`SyntaxOverrides`] consulted
613/// before the built-in field mapping.
614pub fn checked_grammar_from_schema_with(
615    schema: &NodeSchema,
616    ids: &IdGen,
617    overrides: &SyntaxOverrides,
618) -> Result<Grammar, BuildError> {
619    let grammar = grammar_from_schema_with(schema, ids, overrides)?;
620    let mut diags = grammar_check::check_left_recursion(&grammar);
621    diags.extend(grammar_check::check_nullable_repeat(&grammar));
622    diags.extend(
623        grammar_check::check_schema_consistency(&grammar, schema)
624            .into_iter()
625            .filter(|d| d.severity == crate::Severity::Error),
626    );
627    if diags.is_empty() {
628        Ok(grammar)
629    } else {
630        Err(BuildError::new(diags))
631    }
632}
633
634#[cfg(test)]
635mod tests {
636    use super::*;
637    use crate::{RawValue, check_conformance};
638    use dsl_kit_schema::ChildValueShape;
639
640    /// Expr-flavoured schema exercising every supported shape: int /
641    /// string / bool fields, One / Optional / Many children, and a
642    /// zero-argument variant.
643    fn demo_schema() -> NodeSchema {
644        NodeSchema {
645            name: "Expr".into(),
646            variants: vec![
647                VariantSchema {
648                    name: "Lit".into(),
649                    fields: vec![FieldSchema {
650                        name: "value".into(),
651                        ty: "i64".into(),
652                        optional: false,
653                    }],
654                    children: vec![],
655                },
656                VariantSchema {
657                    name: "Name".into(),
658                    fields: vec![
659                        FieldSchema {
660                            name: "text".into(),
661                            ty: "String".into(),
662                            optional: false,
663                        },
664                        FieldSchema {
665                            name: "quoted".into(),
666                            ty: "bool".into(),
667                            optional: false,
668                        },
669                    ],
670                    children: vec![],
671                },
672                VariantSchema {
673                    name: "Add".into(),
674                    fields: vec![],
675                    children: vec![
676                        ChildSchema {
677                            name: "lhs".into(),
678                            multiplicity: Multiplicity::One,
679                            value_shape: ChildValueShape::Recursive,
680                        },
681                        ChildSchema {
682                            name: "rhs".into(),
683                            multiplicity: Multiplicity::One,
684                            value_shape: ChildValueShape::Recursive,
685                        },
686                    ],
687                },
688                VariantSchema {
689                    name: "Neg".into(),
690                    fields: vec![],
691                    children: vec![ChildSchema {
692                        name: "body".into(),
693                        multiplicity: Multiplicity::Optional,
694                        value_shape: ChildValueShape::Recursive,
695                    }],
696                },
697                VariantSchema {
698                    name: "List".into(),
699                    fields: vec![],
700                    children: vec![ChildSchema {
701                        name: "items".into(),
702                        multiplicity: Multiplicity::Many,
703                        value_shape: ChildValueShape::Recursive,
704                    }],
705                },
706                VariantSchema {
707                    name: "Env".into(),
708                    fields: vec![],
709                    children: vec![ChildSchema {
710                        name: "entries".into(),
711                        multiplicity: Multiplicity::Map,
712                        value_shape: ChildValueShape::Recursive,
713                    }],
714                },
715                VariantSchema {
716                    name: "Unit".into(),
717                    fields: vec![],
718                    children: vec![],
719                },
720            ],
721        }
722    }
723
724    fn demo_grammar() -> Grammar {
725        checked_grammar_from_schema(&demo_schema(), &IdGen::new())
726            .expect("demo schema generates a clean grammar")
727    }
728
729    /// Parse + conformance-check against the demo schema in one go.
730    fn parse_ok(input: &str) -> crate::ParseTree {
731        let g = demo_grammar();
732        let tree = g
733            .parse(input)
734            .unwrap_or_else(|e| panic!("parse failed for {input:?}: {:?}", e.diagnostics));
735        let diags = check_conformance(&tree, &demo_schema());
736        assert!(
737            diags.is_empty(),
738            "conformance clean for {input:?}: {diags:?}"
739        );
740        tree
741    }
742
743    #[test]
744    fn generated_grammar_passes_all_static_checks() {
745        // checked_grammar_from_schema already runs the three checks;
746        // this test pins that it succeeds for the full shape matrix.
747        demo_grammar();
748    }
749
750    #[test]
751    fn parses_int_field_variant() {
752        let tree = parse_ok("Lit(value: -42)");
753        assert_eq!(tree.variant, "Lit");
754        assert_eq!(tree.field("value"), Some(&RawValue::Text("-42".into())));
755    }
756
757    #[test]
758    fn parses_string_and_bool_fields_with_escapes() {
759        let tree = parse_ok(r#"Name(text: "hello \"world\"\n", quoted: true)"#);
760        assert_eq!(
761            tree.field("text"),
762            Some(&RawValue::Text("hello \"world\"\n".into()))
763        );
764        assert_eq!(tree.field("quoted"), Some(&RawValue::Text("true".into())));
765    }
766
767    #[test]
768    fn parses_nested_one_children() {
769        let tree = parse_ok("Add(lhs: Lit(value: 1), rhs: Neg(body: none))");
770        let lhs = tree.child_slot("lhs").unwrap();
771        assert_eq!(lhs.len(), 1);
772        assert_eq!(lhs[0].variant, "Lit");
773        let rhs = tree.child_slot("rhs").unwrap();
774        assert_eq!(rhs[0].variant, "Neg");
775    }
776
777    #[test]
778    fn optional_child_none_leaves_slot_absent() {
779        let tree = parse_ok("Neg(body: none)");
780        assert!(
781            tree.child_slot("body").is_none(),
782            "bare `none` = absent slot"
783        );
784    }
785
786    #[test]
787    fn optional_child_present_binds_one_tree() {
788        let tree = parse_ok("Neg(body: Lit(value: 3))");
789        assert_eq!(tree.child_slot("body").unwrap().len(), 1);
790    }
791
792    #[test]
793    fn many_child_list_binds_in_order_and_drops_commas() {
794        let tree = parse_ok("List(items: [Lit(value: 1), Lit(value: 2), Unit()])");
795        let items = tree.child_slot("items").unwrap();
796        let variants: Vec<&str> = items.iter().map(|t| t.variant.as_str()).collect();
797        assert_eq!(variants, ["Lit", "Lit", "Unit"]);
798    }
799
800    #[test]
801    fn many_child_empty_list_is_conformant() {
802        let tree = parse_ok("List(items: [])");
803        assert!(
804            tree.child_slot("items").is_none_or(<[_]>::is_empty),
805            "empty list = zero children"
806        );
807    }
808
809    #[test]
810    fn keyed_map_binds_entries_sorted_by_key() {
811        // Written deliberately out of order, with one quoted key that
812        // could not be spelled as a bare identifier: the parser is
813        // responsible for landing both in canonical order.
814        let tree = parse_ok(r#"Env(entries: { zeta: Unit(), "a b": Lit(value: 1) })"#);
815        let entries = tree.keyed_child_slot("entries").expect("keyed slot bound");
816        let keys: Vec<&str> = entries.iter().map(|(k, _)| k.as_str()).collect();
817        assert_eq!(keys, ["a b", "zeta"]);
818        let variants: Vec<&str> = entries.iter().map(|(_, t)| t.variant.as_str()).collect();
819        assert_eq!(variants, ["Lit", "Unit"]);
820        assert!(
821            tree.children.is_empty(),
822            "keyed entries must not leak into the positional half: {:?}",
823            tree.children
824        );
825    }
826
827    #[test]
828    fn keyed_map_source_order_does_not_change_the_tree() {
829        let forward = parse_ok("Env(entries: { a: Unit(), b: Lit(value: 1) })");
830        let reversed = parse_ok("Env(entries: { b: Lit(value: 1), a: Unit() })");
831        // Spans differ (the two spellings put the subtrees at
832        // different offsets), so compare the keyed payload rather than
833        // the whole tree.
834        let keys = |t: &crate::ParseTree| -> Vec<String> {
835            t.keyed_child_slot("entries")
836                .unwrap()
837                .iter()
838                .map(|(k, v)| format!("{k}={}", v.variant))
839                .collect()
840        };
841        assert_eq!(keys(&forward), keys(&reversed));
842    }
843
844    #[test]
845    fn keyed_map_empty_is_conformant() {
846        let tree = parse_ok("Env(entries: {})");
847        assert!(
848            tree.keyed_child_slot("entries").is_none_or(<[_]>::is_empty),
849            "empty map = zero entries"
850        );
851    }
852
853    #[test]
854    fn keyed_map_duplicate_key_parses_but_fails_conformance() {
855        // The grammar has no reason to reject it — duplicate detection
856        // is the schema's job, and `parse_ok` would hide the split, so
857        // this test drives the two stages apart deliberately.
858        let g = demo_grammar();
859        let tree = g
860            .parse("Env(entries: { k: Unit(), k: Lit(value: 1) })")
861            .expect("grammar accepts a repeated key");
862        let diags = check_conformance(&tree, &demo_schema());
863        assert!(
864            diags.iter().any(|d| d.code == crate::codes::DUPLICATE_KEY),
865            "expected DUPLICATE_KEY; got {:?}",
866            diags.iter().map(|d| d.code.as_str()).collect::<Vec<_>>()
867        );
868    }
869
870    #[test]
871    fn zero_argument_variant_parses() {
872        let tree = parse_ok("Unit()");
873        assert_eq!(tree.variant, "Unit");
874        assert!(tree.fields.is_empty() && tree.children.is_empty());
875    }
876
877    #[test]
878    fn whitespace_and_newlines_are_tolerated() {
879        let tree = parse_ok("Add(\n  lhs: Lit( value : 7 ),\n  rhs: Unit()\n)");
880        assert_eq!(tree.child_slot("lhs").unwrap()[0].variant, "Lit");
881    }
882
883    #[test]
884    fn unsupported_field_type_fails_generation_with_full_list() {
885        let schema = NodeSchema {
886            name: "Bad".into(),
887            variants: vec![VariantSchema {
888                name: "Par".into(),
889                fields: vec![
890                    FieldSchema {
891                        name: "policy".into(),
892                        ty: "Option<JoinPolicy>".into(),
893                        optional: false,
894                    },
895                    FieldSchema {
896                        name: "reducer".into(),
897                        ty: "ReducerId".into(),
898                        optional: false,
899                    },
900                ],
901                children: vec![],
902            }],
903        };
904        let err = grammar_from_schema(&schema, &IdGen::new()).unwrap_err();
905        assert_eq!(err.diagnostics.len(), 2, "one diagnostic per bad field");
906        assert!(
907            err.diagnostics
908                .iter()
909                .all(|d| d.code == codes::UNSUPPORTED_FIELD)
910        );
911    }
912
913    /// A scalar-valued keyed slot is recognised by the derive and
914    /// the JSON front-end, but the canonical text syntax has no
915    /// production for it yet — `grammar_from_schema` must fail
916    /// loudly with [`codes::UNSUPPORTED_MAP_VALUE_SHAPE`] rather
917    /// than silently emit a grammar that expects each entry's value
918    /// to be a full AST node. Pins the pre-flight guard so a
919    /// downstream feeding a Shape 1 schema through PEG generation
920    /// gets a clear pointer to the JSON front-end instead of an
921    /// opaque parse failure.
922    #[test]
923    fn scalar_keyed_slot_fails_grammar_generation_loudly() {
924        let schema = NodeSchema {
925            name: "Cfg".into(),
926            variants: vec![VariantSchema {
927                name: "Env".into(),
928                fields: vec![],
929                children: vec![ChildSchema::scalar_map("entries", "String")],
930            }],
931        };
932        let err = grammar_from_schema(&schema, &IdGen::new()).unwrap_err();
933        assert_eq!(err.diagnostics.len(), 1);
934        assert_eq!(err.diagnostics[0].code, codes::UNSUPPORTED_MAP_VALUE_SHAPE);
935        assert!(
936            err.diagnostics[0].message.contains("entries"),
937            "diagnostic mentions the slot name: {}",
938            err.diagnostics[0].message
939        );
940        assert!(
941            err.diagnostics[0].message.contains("String"),
942            "diagnostic mentions the scalar value type: {}",
943            err.diagnostics[0].message
944        );
945    }
946
947    /// Recursive keyed slots still generate cleanly — the scalar
948    /// gate must not accidentally reject the pre-0.6 keyed shape.
949    #[test]
950    fn recursive_keyed_slot_still_generates_grammar() {
951        let schema = NodeSchema {
952            name: "Cfg".into(),
953            variants: vec![
954                VariantSchema {
955                    name: "Env".into(),
956                    fields: vec![],
957                    children: vec![ChildSchema::recursive("entries", Multiplicity::Map)],
958                },
959                VariantSchema {
960                    name: "Unit".into(),
961                    fields: vec![],
962                    children: vec![],
963                },
964            ],
965        };
966        grammar_from_schema(&schema, &IdGen::new())
967            .expect("recursive keyed shape still supported end-to-end in the text front-end");
968    }
969
970    /// Flow-shaped schema whose `Par` payload fields the built-in
971    /// mapping rejects — the override motivating case.
972    fn par_schema() -> NodeSchema {
973        NodeSchema {
974            name: "Flow".into(),
975            variants: vec![
976                VariantSchema {
977                    name: "Par".into(),
978                    fields: vec![
979                        FieldSchema {
980                            name: "policy".into(),
981                            // Token-stream spelling, as the derive extracts it.
982                            ty: "Option < JoinPolicy >".into(),
983                            optional: false,
984                        },
985                        FieldSchema {
986                            name: "reducer_id".into(),
987                            ty: "Option < String >".into(),
988                            optional: false,
989                        },
990                    ],
991                    children: vec![ChildSchema {
992                        name: "children".into(),
993                        multiplicity: Multiplicity::Many,
994                        value_shape: ChildValueShape::Recursive,
995                    }],
996                },
997                VariantSchema {
998                    name: "Call".into(),
999                    fields: vec![FieldSchema {
1000                        name: "label".into(),
1001                        ty: "String".into(),
1002                        optional: false,
1003                    }],
1004                    children: vec![],
1005                },
1006            ],
1007        }
1008    }
1009
1010    fn par_overrides() -> SyntaxOverrides {
1011        SyntaxOverrides::new()
1012            .for_type("Option<JoinPolicy>", |ids| {
1013                choice(
1014                    ids,
1015                    vec![
1016                        token(ids, "%kw:none"),
1017                        token(ids, "%kw:all_failfast"),
1018                        token(ids, "%kw:any_failfast"),
1019                    ],
1020                )
1021            })
1022            .for_type("Option<String>", |ids| {
1023                choice(ids, vec![token(ids, "%kw:none"), token(ids, "%str")])
1024            })
1025    }
1026
1027    #[test]
1028    fn type_overrides_unlock_exotic_payload_fields() {
1029        // Without overrides the Flow-shaped schema fails generation
1030        // (pinned by unsupported_field_type_fails_generation_with_full_list);
1031        // with them it generates check-clean and parses. Note the
1032        // override keys are written without whitespace while the schema
1033        // carries the token-stream spelling.
1034        let g = checked_grammar_from_schema_with(&par_schema(), &IdGen::new(), &par_overrides())
1035            .expect("overridden Flow schema generates a clean grammar");
1036        let tree = g
1037            .parse(
1038                r#"Par(policy: all_failfast, reducer_id: "reduce_all_ordered",
1039                     children: [Call(label: "a"), Call(label: "b")])"#,
1040            )
1041            .expect("canonical Par text parses");
1042        let diags = check_conformance(&tree, &par_schema());
1043        assert!(diags.is_empty(), "conformance clean: {diags:?}");
1044        assert_eq!(
1045            tree.field("policy"),
1046            Some(&RawValue::Text("all_failfast".into()))
1047        );
1048        assert_eq!(
1049            tree.field("reducer_id"),
1050            Some(&RawValue::Text("reduce_all_ordered".into()))
1051        );
1052        assert_eq!(tree.child_slot("children").unwrap().len(), 2);
1053    }
1054
1055    #[test]
1056    fn field_override_wins_over_type_override() {
1057        // Same type in both entries; the (variant, field)-pinned one
1058        // must decide the syntax.
1059        let overrides = par_overrides().for_field("Par", "policy", |ids| token(ids, "%kw:pinned"));
1060        let g = checked_grammar_from_schema_with(&par_schema(), &IdGen::new(), &overrides)
1061            .expect("clean grammar with field-level override");
1062        let tree = g
1063            .parse(r#"Par(policy: pinned, reducer_id: none, children: [])"#)
1064            .expect("field-level syntax parses");
1065        assert_eq!(tree.field("policy"), Some(&RawValue::Text("pinned".into())));
1066        // The type-level spelling is no longer accepted at that site.
1067        assert!(
1068            g.parse(r#"Par(policy: all_failfast, reducer_id: none, children: [])"#)
1069                .is_err(),
1070            "type-level syntax rejected once the field is pinned"
1071        );
1072    }
1073
1074    #[test]
1075    fn unrelated_override_leaves_builtin_mapping_and_failure_intact() {
1076        // An override for some other type neither rescues the
1077        // unsupported fields nor perturbs built-in ones. Note that
1078        // `Option<String>` on `reducer_id` is now a built-in mapping
1079        // (Layer 2 of the "built-in optional payload fields"
1080        // contract), so only the `Option<JoinPolicy>` field on
1081        // `policy` remains unmapped without a `SyntaxOverrides` entry.
1082        let overrides = SyntaxOverrides::new().for_type("Uuid", |ids| token(ids, "%str"));
1083        let err = grammar_from_schema_with(&par_schema(), &IdGen::new(), &overrides).unwrap_err();
1084        assert_eq!(
1085            err.diagnostics.len(),
1086            1,
1087            "policy is the only Par field still unmapped without overrides",
1088        );
1089        assert!(
1090            err.diagnostics
1091                .iter()
1092                .all(|d| d.code == codes::UNSUPPORTED_FIELD)
1093        );
1094        assert!(
1095            err.diagnostics[0].message.contains("policy"),
1096            "unmapped-field diagnostic points at `policy`",
1097        );
1098        // Built-in demo schema is unaffected by the stray entry.
1099        let g = checked_grammar_from_schema_with(&demo_schema(), &IdGen::new(), &overrides)
1100            .expect("demo schema still generates");
1101        assert_eq!(g.parse("Lit(value: 7)").unwrap().variant, "Lit");
1102    }
1103
1104    /// Layer 2: `Option<String>` payload maps to `none | %str` via the
1105    /// built-in mapping (no `SyntaxOverrides` entry needed). Canonical
1106    /// text parses both spellings and both build through the derive's
1107    /// `build_field_optional` route.
1108    #[test]
1109    fn builtin_option_string_mapping_accepts_none_and_str() {
1110        let schema = NodeSchema {
1111            name: "Meta".into(),
1112            variants: vec![VariantSchema {
1113                name: "Row".into(),
1114                fields: vec![FieldSchema {
1115                    name: "description".into(),
1116                    ty: "Option<String>".into(),
1117                    optional: true,
1118                }],
1119                children: vec![],
1120            }],
1121        };
1122        let g = checked_grammar_from_schema(&schema, &IdGen::new())
1123            .expect("Option<String> is a built-in mapping");
1124        // `none` spelling.
1125        let t1 = g.parse(r#"Row(description: none)"#).expect("none parses");
1126        assert_eq!(
1127            t1.field("description"),
1128            Some(&RawValue::Text("none".into())),
1129        );
1130        // `%str` spelling.
1131        let t2 = g.parse(r#"Row(description: "hi")"#).expect("string parses");
1132        assert_eq!(t2.field("description"), Some(&RawValue::Text("hi".into())));
1133        // Omitted entirely (Layer 1 optional PEG wrap).
1134        let t3 = g.parse(r#"Row()"#).expect("omitted parses");
1135        assert!(t3.field("description").is_none());
1136        // check_conformance is clean for all three.
1137        for t in [&t1, &t2, &t3] {
1138            assert!(
1139                check_conformance(t, &schema).is_empty(),
1140                "conformance clean for {t:?}",
1141            );
1142        }
1143    }
1144
1145    /// Layer 2: `Vec<String>` payload maps to `[ %str ("," %str)* ]`
1146    /// via the built-in mapping. Empty list, single item, and multi
1147    /// item all parse; captured text is JSON-parsable so
1148    /// `build_field_vec` reads it directly.
1149    #[test]
1150    fn builtin_vec_string_mapping_accepts_bracketed_list() {
1151        let schema = NodeSchema {
1152            name: "Meta".into(),
1153            variants: vec![VariantSchema {
1154                name: "Row".into(),
1155                fields: vec![FieldSchema {
1156                    name: "tags".into(),
1157                    ty: "Vec<String>".into(),
1158                    optional: true,
1159                }],
1160                children: vec![],
1161            }],
1162        };
1163        let g = checked_grammar_from_schema(&schema, &IdGen::new())
1164            .expect("Vec<String> is a built-in mapping");
1165        // Empty list.
1166        let t1 = g.parse(r#"Row(tags: [])"#).expect("empty list parses");
1167        assert_eq!(t1.field("tags"), Some(&RawValue::Text("[]".into())));
1168        // Multi item. The PEG runtime's implicit whitespace-skip
1169        // strips inter-token spaces from the captured text, so the
1170        // field carries the compact form `["a","b"]` (still valid
1171        // JSON — `serde_json::from_str` accepts it).
1172        let t2 = g
1173            .parse(r#"Row(tags: ["a", "b"])"#)
1174            .expect("two items parses");
1175        assert_eq!(
1176            t2.field("tags"),
1177            Some(&RawValue::Text(r#"["a","b"]"#.into())),
1178        );
1179        // Omitted.
1180        let t3 = g.parse(r#"Row()"#).expect("omitted parses");
1181        assert!(t3.field("tags").is_none());
1182        for t in [&t1, &t2, &t3] {
1183            assert!(
1184                check_conformance(t, &schema).is_empty(),
1185                "conformance clean for {t:?}",
1186            );
1187        }
1188    }
1189
1190    #[test]
1191    fn empty_schema_fails_generation() {
1192        let schema = NodeSchema {
1193            name: "Void".into(),
1194            variants: vec![],
1195        };
1196        let err = grammar_from_schema(&schema, &IdGen::new()).unwrap_err();
1197        assert_eq!(err.diagnostics[0].code, codes::EMPTY_SCHEMA);
1198    }
1199
1200    #[test]
1201    fn parse_error_reports_expected_set() {
1202        let g = demo_grammar();
1203        let err = g.parse("Lit(value: banana)").unwrap_err();
1204        assert!(!err.diagnostics.is_empty());
1205    }
1206}