dsl_kit_parse/schema_gen.rs
1//! Schema-driven grammar generation (item N).
2//!
3//! The kit's identity axis is "write the AST in Rust, the toolchain
4//! supplies the rest". [`grammar_from_schema`] converts the type-level
5//! shape a `#[derive(DslSchema)]` already extracts into a working
6//! [`Grammar`] for a canonical text syntax — the DSL author writes two
7//! derives and gets an external-format parser for free, instead of
8//! hand-authoring a grammar (in Rust or, worse, in a Bison-style JSON
9//! blob nobody wants to write).
10//!
11//! # Canonical syntax
12//!
13//! One form per variant: the variant name as a keyword, followed by a
14//! parenthesised, comma-separated argument list. Arguments appear in
15//! schema order — payload fields first, then child slots — each spelled
16//! `name: value`:
17//!
18//! ```text
19//! Add(lhs: Lit(value: 1), rhs: Neg(body: none))
20//! List(items: [Lit(value: 1), Lit(value: 2)])
21//! Name(text: "hello world")
22//! Unit()
23//! ```
24//!
25//! - **`One` child** — `name: <node>`.
26//! - **`Optional` child** — `name: <node>` or `name: none`. (If a
27//! variant is literally called `none`, PEG ordered choice still
28//! parses `name: none(...)` as that variant; bare `none` means
29//! absent.)
30//! - **`Many` child** — `name: [<node>, <node>, ...]`, empty list
31//! allowed.
32//! - **`Map` child** — `name: { <key>: <node>, <key>: <node> }`, empty
33//! map allowed. Braces (not the `Many` brackets) mark the slot as
34//! keyed, matching the JSON bridge's object spelling. A key is
35//! either a `%str` literal or a bare `%ident`, so `"a b": …` and
36//! `path: …` are both writable. Entries may be written in any
37//! order — the parser sorts them into the canonical
38//! ascending-by-key form the `ParseTree` requires.
39//! - **Payload fields** — by Rust type source text:
40//! `String` → a `%str` literal; the integer types → `%int`;
41//! `bool` → `true` / `false`. Any other payload type fails
42//! generation with [`codes::UNSUPPORTED_FIELD`] — loudly, per
43//! variant, rather than silently dropping the variant from the
44//! grammar — unless a [`SyntaxOverrides`] entry supplies its value
45//! production.
46//!
47//! # Syntax overrides
48//!
49//! [`grammar_from_schema_with`] accepts a [`SyntaxOverrides`] carrying
50//! value productions for payload types the built-in mapping rejects
51//! (e.g. `Option<JoinPolicy>`), keyed by type source text or pinned to
52//! one `(variant, field)` site. The override supplies only the value
53//! [`Peg`] — the surrounding `name: value` argument shape stays
54//! canonical — and its matched text becomes the field's production, so
55//! the override author owns making that text consumable downstream
56//! (typically `FromStr` on the field type, via
57//! [`build_field`](crate::build_field)).
58//!
59//! # Guarantees
60//!
61//! The generated grammar is left-recursion-free by construction (every
62//! variant rule starts with its keyword token) and contains no nullable
63//! repeats; `GrammarCheck`'s `check_left_recursion` /
64//! `check_nullable_repeat` / `check_schema_consistency` all pass
65//! against it — the tests pin this. Trees produced by parsing satisfy
66//! [`check_conformance`](crate::check_conformance) against the source
67//! schema.
68
69use std::collections::HashMap;
70
71use crate::grammar_check;
72use crate::peg::{
73 Grammar, Peg, choice, field, keyed_entry, node, repeat, rule, rule_ref, seq, token,
74};
75use crate::{BuildError, Diagnostic};
76use dsl_kit_core::IdGen;
77use dsl_kit_schema::{
78 ChildSchema, ChildValueShape, FieldSchema, Multiplicity, NodeSchema, VariantSchema,
79};
80
81/// Diagnostic codes emitted by grammar generation.
82pub mod codes {
83 /// A payload field's Rust type has no canonical-syntax mapping.
84 pub const UNSUPPORTED_FIELD: &str = "dsl_kit::schema_gen::unsupported_field";
85 /// The schema declares no variants — there is nothing to parse.
86 pub const EMPTY_SCHEMA: &str = "dsl_kit::schema_gen::empty_schema";
87 /// A child slot's multiplicity is a variant this build of
88 /// `dsl-kit-parse` does not recognise (added to the schema crate's
89 /// `#[non_exhaustive]` enum ahead of parse-side support). Emitted
90 /// by the pre-flight check in
91 /// [`grammar_from_schema_with`] so `child_arg_peg` never sees an
92 /// unknown variant.
93 pub const UNKNOWN_MULTIPLICITY: &str = "dsl_kit::schema_gen::unknown_multiplicity";
94 /// A keyed child slot declares a value shape the canonical text
95 /// syntax has not yet grown a production for. Emitted by the
96 /// pre-flight check so a scalar-valued keyed slot fails loudly
97 /// at grammar-generation time rather than silently producing a
98 /// grammar that expects each entry's value to be a full AST
99 /// node — the shape mismatch would otherwise surface as an
100 /// opaque parse failure on the DSL author's first document.
101 pub const UNSUPPORTED_MAP_VALUE_SHAPE: &str =
102 "dsl_kit::schema_gen::unsupported_map_value_shape";
103}
104
105/// Name of the generated start rule.
106pub const START_RULE: &str = "node";
107
108/// A deferred value-production constructor: overrides are registered
109/// before the [`IdGen`] doing the generation exists, so each entry is a
110/// factory invoked with the generator's `ids` at build time.
111type PegFactory = Box<dyn Fn(&IdGen) -> Peg>;
112
113/// Value-production overrides for payload fields whose Rust type has no
114/// built-in canonical-syntax mapping (see the module docs).
115///
116/// Two granularities, resolved most-specific-first:
117///
118/// 1. **Per field** ([`Self::for_field`]) — pinned to one
119/// `(variant, field)` site.
120/// 2. **Per type** ([`Self::for_type`]) — keyed by the field's Rust
121/// type source text; whitespace is ignored, so
122/// `"Option<JoinPolicy>"` matches the derive-extracted
123/// `"Option < JoinPolicy >"` spelling.
124///
125/// A field with neither entry falls back to the built-in mapping, and
126/// failing that still fails generation loudly with
127/// [`codes::UNSUPPORTED_FIELD`].
128///
129/// The factory builds only the **value** production; the enclosing
130/// `name ':' <value>` argument shape (and the `Field` sink binding the
131/// matched text to the field name) stays canonical, which keeps the
132/// generated grammar's `GrammarCheck` guarantees intact.
133#[derive(Default)]
134pub struct SyntaxOverrides {
135 by_type: HashMap<String, PegFactory>,
136 by_field: HashMap<(String, String), PegFactory>,
137}
138
139impl SyntaxOverrides {
140 /// An empty override set (equivalent to the built-in mapping only).
141 pub fn new() -> Self {
142 Self::default()
143 }
144
145 /// Registers a value production for every payload field of Rust
146 /// type `ty` (source-text comparison, whitespace-insensitive).
147 pub fn for_type(
148 mut self,
149 ty: impl AsRef<str>,
150 value: impl Fn(&IdGen) -> Peg + 'static,
151 ) -> Self {
152 self.by_type.insert(strip_ws(ty.as_ref()), Box::new(value));
153 self
154 }
155
156 /// Registers a value production for the single field `field` of
157 /// variant `variant`, winning over any [`Self::for_type`] entry.
158 pub fn for_field(
159 mut self,
160 variant: impl Into<String>,
161 field: impl Into<String>,
162 value: impl Fn(&IdGen) -> Peg + 'static,
163 ) -> Self {
164 self.by_field
165 .insert((variant.into(), field.into()), Box::new(value));
166 self
167 }
168
169 /// Most-specific matching factory for `field` of `variant`, if any.
170 fn resolve(&self, variant: &str, field: &FieldSchema) -> Option<&PegFactory> {
171 self.by_field
172 .get(&(variant.to_string(), field.name.clone()))
173 .or_else(|| self.by_type.get(&strip_ws(&field.ty)))
174 }
175}
176
177/// Whitespace-insensitive type key: `Option < JoinPolicy >` (the
178/// token-stream spelling `#[derive(DslSchema)]` extracts) and
179/// `Option<JoinPolicy>` (what an author writes) must compare equal.
180fn strip_ws(ty: &str) -> String {
181 ty.chars().filter(|c| !c.is_whitespace()).collect()
182}
183
184impl Grammar {
185 /// Generates the canonical-syntax grammar for `schema`. See
186 /// [`grammar_from_schema`].
187 pub fn from_schema(schema: &NodeSchema, ids: &IdGen) -> Result<Grammar, BuildError> {
188 grammar_from_schema(schema, ids)
189 }
190}
191
192/// Generates a [`Grammar`] for the canonical named-argument syntax of
193/// `schema` (see the module docs for the syntax), built-in field
194/// mapping only.
195///
196/// Fails with one [`codes::UNSUPPORTED_FIELD`] diagnostic per
197/// unmappable payload field (collected across all variants, so the
198/// author sees the full list at once). To map such fields instead of
199/// failing, use [`grammar_from_schema_with`].
200pub fn grammar_from_schema(schema: &NodeSchema, ids: &IdGen) -> Result<Grammar, BuildError> {
201 grammar_from_schema_with(schema, ids, &SyntaxOverrides::default())
202}
203
204/// [`grammar_from_schema`] with [`SyntaxOverrides`] consulted before
205/// the built-in field mapping (per-field entries win over per-type
206/// entries win over built-ins).
207pub fn grammar_from_schema_with(
208 schema: &NodeSchema,
209 ids: &IdGen,
210 overrides: &SyntaxOverrides,
211) -> Result<Grammar, BuildError> {
212 if schema.variants.is_empty() {
213 return Err(BuildError::single(Diagnostic::error(
214 codes::EMPTY_SCHEMA,
215 format!("schema `{}` declares no variants", schema.name),
216 )));
217 }
218
219 // Pre-flight: collect every diagnostic that must block rule
220 // generation, then return in one shot. A single pass means an
221 // author with both a bad payload type and a `Multiplicity::Map`
222 // slot sees both errors on the first build instead of paying two
223 // edit-compile cycles for one report.
224 //
225 // Grouping:
226 // - `UNSUPPORTED_FIELD` for payload types with no canonical-syntax
227 // mapping (and no `SyntaxOverrides` entry).
228 // - `UNKNOWN_MULTIPLICITY` for child slots whose multiplicity
229 // this build does not know, which is what keeps
230 // `child_arg_peg`'s catch-all arm from ever running.
231 let mut pre_flight = Vec::new();
232 for v in &schema.variants {
233 for f in &v.fields {
234 if resolved_field_value_peg(&v.name, f, ids, overrides).is_none() {
235 pre_flight.push(Diagnostic::error(
236 codes::UNSUPPORTED_FIELD,
237 format!(
238 "variant `{}` field `{}`: type `{}` has no canonical-syntax \
239 mapping (supported: String, bool, the integer types; or \
240 register a `SyntaxOverrides` value production)",
241 v.name, f.name, f.ty
242 ),
243 ));
244 }
245 }
246 for c in &v.children {
247 match c.multiplicity {
248 // Every shape `child_arg_peg` knows how to emit.
249 Multiplicity::One | Multiplicity::Optional | Multiplicity::Many => {}
250 // Keyed slots come in two value shapes at derive
251 // level: recursive (`BTreeMap<String, Self>` /
252 // `BTreeMap<String, Box<Self>>`) is supported by the
253 // canonical text syntax; scalar
254 // (`ChildValueShape::Scalar { .. }`, Shape 1 of the
255 // tracking issue) has not grown a text production
256 // yet — `child_arg_peg` would emit
257 // `key: <node>` for every entry, producing an
258 // unusable grammar. Fail loudly instead so the DSL
259 // author is directed to the JSON front-end.
260 Multiplicity::Map => {
261 if let ChildValueShape::Scalar { ty } = &c.value_shape {
262 pre_flight.push(Diagnostic::error(
263 codes::UNSUPPORTED_MAP_VALUE_SHAPE,
264 format!(
265 "variant `{}` child slot `{}`: keyed scalar values \
266 (`BTreeMap<String, {}>`) are recognised by the derive and \
267 the JSON front-end, but the canonical text syntax has no \
268 production for them yet — use the JSON front-end for \
269 scalar-map DSLs, or open the tracking issue if the text \
270 syntax is needed",
271 v.name, c.name, ty
272 ),
273 ));
274 }
275 }
276 // `#[non_exhaustive]` catch-all: a future Multiplicity
277 // variant added to dsl-kit-schema before this crate is
278 // updated must also be rejected here — that's what
279 // keeps `child_arg_peg`'s `_ => unreachable!()` arm
280 // honest.
281 _ => {
282 pre_flight.push(Diagnostic::error(
283 codes::UNKNOWN_MULTIPLICITY,
284 format!(
285 "variant `{}` child slot `{}`: unrecognised Multiplicity \
286 variant (upgrade dsl-kit-parse to a version that knows about it)",
287 v.name, c.name
288 ),
289 ));
290 }
291 }
292 }
293 }
294 if !pre_flight.is_empty() {
295 return Err(BuildError::new(pre_flight));
296 }
297
298 let mut rules = Vec::with_capacity(schema.variants.len() + 1);
299 rules.push(rule(
300 ids,
301 START_RULE,
302 choice(
303 ids,
304 schema
305 .variants
306 .iter()
307 .map(|v| rule_ref(ids, v.name.clone()))
308 .collect(),
309 ),
310 ));
311 for v in &schema.variants {
312 rules.push(variant_rule(v, ids, overrides));
313 }
314 Ok(Grammar::new(rules, START_RULE))
315}
316
317/// Builds the rule for one variant. Two shapes:
318///
319/// - **Strict-order** (default, variant has no optional fields):
320/// `%kw:V "(" arg ("," arg)* ")"` with arguments emitted in schema
321/// order (fields, then children). Preserves the pre-0.3 canonical
322/// spelling for variants that never omit.
323/// - **Free-order** (variant has ≥ 1 optional field): `%kw:V "(" (arg
324/// ("," arg)*)? ")"` where each arg is `arg_1 | arg_2 | ...` and
325/// every alternative carries its own `%kw:name` prefix. Args are
326/// distinguishable by that name-keyword, so the DSL author may
327/// omit any subset of optional arguments (or even all of them),
328/// and `check_conformance` remains the authority on required /
329/// duplicate / unknown-slot diagnostics.
330fn variant_rule(v: &VariantSchema, ids: &IdGen, overrides: &SyntaxOverrides) -> Peg {
331 let has_optional = v.fields.iter().any(|f| f.optional);
332 let mut args: Vec<Peg> = Vec::new();
333 for f in &v.fields {
334 let value = resolved_field_value_peg(&v.name, f, ids, overrides)
335 .expect("unsupported field types were rejected before rule generation");
336 args.push(seq(
337 ids,
338 vec![
339 token(ids, format!("%kw:{}", f.name)),
340 token(ids, ":"),
341 field(ids, f.name.clone(), value),
342 ],
343 ));
344 }
345 for c in &v.children {
346 args.push(child_arg_peg(c, ids));
347 }
348
349 let mut items = vec![token(ids, format!("%kw:{}", v.name)), token(ids, "(")];
350 if has_optional && !args.is_empty() {
351 // Free-order: any of the argument shapes, comma-separated,
352 // whole list optional. Each argument-slot is a Field capture
353 // in its own right (built above), and the name-keyword prefix
354 // keeps the alternation unambiguous at the byte level.
355 let arg_choice = choice(ids, args);
356 // The tail repeats `("," arg_choice)`; we rebuild arg_choice
357 // per position so each Choice node gets a fresh id (Peg nodes
358 // are id-unique in a grammar).
359 let tail_alt = |ids: &IdGen| -> Peg {
360 let mut alts: Vec<Peg> = Vec::new();
361 for f in &v.fields {
362 let value = resolved_field_value_peg(&v.name, f, ids, overrides)
363 .expect("unsupported field types were rejected before rule generation");
364 alts.push(seq(
365 ids,
366 vec![
367 token(ids, format!("%kw:{}", f.name)),
368 token(ids, ":"),
369 field(ids, f.name.clone(), value),
370 ],
371 ));
372 }
373 for c in &v.children {
374 alts.push(child_arg_peg(c, ids));
375 }
376 choice(ids, alts)
377 };
378 let tail = repeat(ids, seq(ids, vec![token(ids, ","), tail_alt(ids)]), 0, None);
379 let full = seq(ids, vec![arg_choice, tail]);
380 items.push(repeat(ids, full, 0, Some(1)));
381 } else {
382 for (i, arg) in args.into_iter().enumerate() {
383 if i > 0 {
384 items.push(token(ids, ","));
385 }
386 items.push(arg);
387 }
388 }
389 items.push(token(ids, ")"));
390
391 rule(
392 ids,
393 v.name.clone(),
394 node(ids, v.name.clone(), seq(ids, items)),
395 )
396}
397
398/// Value production for a payload field: the most specific
399/// [`SyntaxOverrides`] entry if registered, else the built-in mapping.
400fn resolved_field_value_peg(
401 variant: &str,
402 f: &FieldSchema,
403 ids: &IdGen,
404 overrides: &SyntaxOverrides,
405) -> Option<Peg> {
406 match overrides.resolve(variant, f) {
407 Some(factory) => Some(factory(ids)),
408 None => field_value_peg(f, ids),
409 }
410}
411
412/// Built-in value production for a payload field, by Rust type source
413/// text. `None` when the type has no canonical-syntax mapping.
414///
415/// Recognised (whitespace-insensitive on the type-source spelling):
416///
417/// - `String` → `%str`
418/// - the integer types → `%int`
419/// - `bool` → `true` / `false`
420/// - `Option<String>` → `none` | `%str`
421/// - `Vec<String>` → `[ %str ("," %str)* ]`, empty list allowed
422///
423/// The last two entries realise Layer 2 of the "built-in optional
424/// payload fields" contract — downstream DSLs no longer need to
425/// register a [`SyntaxOverrides`] entry (nor a paired
426/// `#[dsl_build(with = ...)]` converter) for these shapes.
427fn field_value_peg(f: &FieldSchema, ids: &IdGen) -> Option<Peg> {
428 const INT_TYPES: &[&str] = &[
429 "i8", "i16", "i32", "i64", "i128", "u8", "u16", "u32", "u64", "u128", "usize", "isize",
430 ];
431 let ty = strip_ws(&f.ty);
432 if ty == "String" {
433 Some(token(ids, "%str"))
434 } else if INT_TYPES.contains(&ty.as_str()) {
435 Some(token(ids, "%int"))
436 } else if ty == "bool" {
437 Some(choice(
438 ids,
439 vec![token(ids, "%kw:true"), token(ids, "%kw:false")],
440 ))
441 } else if ty == "Option<String>" {
442 // Canonical spelling of an absent Option-payload; mirrors the
443 // `Multiplicity::Optional` child idiom.
444 Some(choice(
445 ids,
446 vec![token(ids, "%kw:none"), token(ids, "%str")],
447 ))
448 } else if ty == "Vec<String>" {
449 // `[ %str_raw ("," %str_raw)* ]` with empty list allowed.
450 // `%str_raw` (peg-level primitive) contributes the raw
451 // source slice for each element — quotes and escapes intact —
452 // so the joined field text is a JSON-compatible array literal
453 // that `build_field_vec` can hand straight to
454 // `serde_json::from_str`.
455 let elems = seq(
456 ids,
457 vec![
458 token(ids, "%str_raw"),
459 repeat(
460 ids,
461 seq(ids, vec![token(ids, ","), token(ids, "%str_raw")]),
462 0,
463 None,
464 ),
465 ],
466 );
467 Some(seq(
468 ids,
469 vec![
470 token(ids, "["),
471 repeat(ids, elems, 0, Some(1)),
472 token(ids, "]"),
473 ],
474 ))
475 } else {
476 None
477 }
478}
479
480/// Argument production for one child slot, by multiplicity.
481fn child_arg_peg(c: &ChildSchema, ids: &IdGen) -> Peg {
482 let name_kw = token(ids, format!("%kw:{}", c.name));
483 let colon = token(ids, ":");
484 match c.multiplicity {
485 Multiplicity::One => seq(
486 ids,
487 vec![
488 name_kw,
489 colon,
490 field(ids, c.name.clone(), rule_ref(ids, START_RULE)),
491 ],
492 ),
493 // `name: <node>` or `name: none`. The Field arm is tried first,
494 // so a variant literally named `none` still wins when followed
495 // by `(`. The bare-`none` arm sits *outside* any Field: its
496 // matched text lands on the enclosing Node sink where it is
497 // dropped as syntactic noise, leaving the slot absent.
498 Multiplicity::Optional => seq(
499 ids,
500 vec![
501 name_kw,
502 colon,
503 choice(
504 ids,
505 vec![
506 field(ids, c.name.clone(), rule_ref(ids, START_RULE)),
507 token(ids, "%kw:none"),
508 ],
509 ),
510 ],
511 ),
512 // `name: [ <node> ("," <node>)* ]`, empty list allowed. The
513 // list commas live inside the Field body; the mixed-field
514 // "trees win" fallback drops them as noise.
515 Multiplicity::Many => {
516 let elems = seq(
517 ids,
518 vec![
519 rule_ref(ids, START_RULE),
520 repeat(
521 ids,
522 seq(ids, vec![token(ids, ","), rule_ref(ids, START_RULE)]),
523 0,
524 None,
525 ),
526 ],
527 );
528 seq(
529 ids,
530 vec![
531 name_kw,
532 colon,
533 token(ids, "["),
534 field(ids, c.name.clone(), repeat(ids, elems, 0, Some(1))),
535 token(ids, "]"),
536 ],
537 )
538 }
539 // `name: { <key>: <node> ("," <key>: <node>)* }`, empty map
540 // allowed. Braces rather than the `Many` idiom's brackets so
541 // the two shapes are distinguishable at a glance, and matching
542 // the JSON bridge's object spelling so a DSL reads the same
543 // way in both front-ends.
544 //
545 // The key token is `%str | %ident`: keys are arbitrary strings
546 // at the `ParseTree` level, so a quoted form has to exist, but
547 // requiring quotes around every `PATH`-style key would be
548 // noise. `%str` is tried first — a quoted key is unambiguous,
549 // while `%ident` cannot start with `"` — and its production is
550 // the decoded content, so `"a b"` and `a` both arrive as plain
551 // key text.
552 Multiplicity::Map => {
553 //
554 // The `:` separator rides on the *value* half, not the
555 // key half: every token matched inside the key production
556 // contributes to the key text, so a `:` there would land
557 // in the key itself. On the value side it is dropped as
558 // syntactic noise, the same way the `Many` list's commas
559 // are.
560 let entry = |ids: &IdGen| -> Peg {
561 keyed_entry(
562 ids,
563 c.name.clone(),
564 choice(ids, vec![token(ids, "%str"), token(ids, "%ident")]),
565 seq(ids, vec![token(ids, ":"), rule_ref(ids, START_RULE)]),
566 )
567 };
568 let entries = seq(
569 ids,
570 vec![
571 entry(ids),
572 repeat(ids, seq(ids, vec![token(ids, ","), entry(ids)]), 0, None),
573 ],
574 );
575 seq(
576 ids,
577 vec![
578 name_kw,
579 colon,
580 token(ids, "{"),
581 repeat(ids, entries, 0, Some(1)),
582 token(ids, "}"),
583 ],
584 )
585 }
586 // `#[non_exhaustive]` catch-all: a future Multiplicity variant
587 // added to dsl-kit-schema before this crate is updated would
588 // land here. The pre-flight check in
589 // `grammar_from_schema_with` should be extended in lockstep to
590 // reject unknown variants; if that extension is missed the
591 // panic here surfaces the omission loudly instead of emitting
592 // a silently-wrong grammar rule.
593 _ => unreachable!(
594 "child_arg_peg encountered an unrecognised Multiplicity variant on slot \
595 `{}` — extend the pre-flight check in grammar_from_schema_with to reject it",
596 c.name
597 ),
598 }
599}
600
601/// Convenience: generate and immediately assert the grammar is clean
602/// under `GrammarCheck` (left recursion, nullable repeats, schema
603/// consistency). Intended for tests and one-shot setup paths; returns
604/// the same grammar as [`grammar_from_schema`].
605pub fn checked_grammar_from_schema(
606 schema: &NodeSchema,
607 ids: &IdGen,
608) -> Result<Grammar, BuildError> {
609 checked_grammar_from_schema_with(schema, ids, &SyntaxOverrides::default())
610}
611
612/// [`checked_grammar_from_schema`] with [`SyntaxOverrides`] consulted
613/// before the built-in field mapping.
614pub fn checked_grammar_from_schema_with(
615 schema: &NodeSchema,
616 ids: &IdGen,
617 overrides: &SyntaxOverrides,
618) -> Result<Grammar, BuildError> {
619 let grammar = grammar_from_schema_with(schema, ids, overrides)?;
620 let mut diags = grammar_check::check_left_recursion(&grammar);
621 diags.extend(grammar_check::check_nullable_repeat(&grammar));
622 diags.extend(
623 grammar_check::check_schema_consistency(&grammar, schema)
624 .into_iter()
625 .filter(|d| d.severity == crate::Severity::Error),
626 );
627 if diags.is_empty() {
628 Ok(grammar)
629 } else {
630 Err(BuildError::new(diags))
631 }
632}
633
634#[cfg(test)]
635mod tests {
636 use super::*;
637 use crate::{RawValue, check_conformance};
638 use dsl_kit_schema::ChildValueShape;
639
640 /// Expr-flavoured schema exercising every supported shape: int /
641 /// string / bool fields, One / Optional / Many children, and a
642 /// zero-argument variant.
643 fn demo_schema() -> NodeSchema {
644 NodeSchema {
645 name: "Expr".into(),
646 variants: vec![
647 VariantSchema {
648 name: "Lit".into(),
649 fields: vec![FieldSchema {
650 name: "value".into(),
651 ty: "i64".into(),
652 optional: false,
653 }],
654 children: vec![],
655 },
656 VariantSchema {
657 name: "Name".into(),
658 fields: vec![
659 FieldSchema {
660 name: "text".into(),
661 ty: "String".into(),
662 optional: false,
663 },
664 FieldSchema {
665 name: "quoted".into(),
666 ty: "bool".into(),
667 optional: false,
668 },
669 ],
670 children: vec![],
671 },
672 VariantSchema {
673 name: "Add".into(),
674 fields: vec![],
675 children: vec![
676 ChildSchema {
677 name: "lhs".into(),
678 multiplicity: Multiplicity::One,
679 value_shape: ChildValueShape::Recursive,
680 },
681 ChildSchema {
682 name: "rhs".into(),
683 multiplicity: Multiplicity::One,
684 value_shape: ChildValueShape::Recursive,
685 },
686 ],
687 },
688 VariantSchema {
689 name: "Neg".into(),
690 fields: vec![],
691 children: vec![ChildSchema {
692 name: "body".into(),
693 multiplicity: Multiplicity::Optional,
694 value_shape: ChildValueShape::Recursive,
695 }],
696 },
697 VariantSchema {
698 name: "List".into(),
699 fields: vec![],
700 children: vec![ChildSchema {
701 name: "items".into(),
702 multiplicity: Multiplicity::Many,
703 value_shape: ChildValueShape::Recursive,
704 }],
705 },
706 VariantSchema {
707 name: "Env".into(),
708 fields: vec![],
709 children: vec![ChildSchema {
710 name: "entries".into(),
711 multiplicity: Multiplicity::Map,
712 value_shape: ChildValueShape::Recursive,
713 }],
714 },
715 VariantSchema {
716 name: "Unit".into(),
717 fields: vec![],
718 children: vec![],
719 },
720 ],
721 }
722 }
723
724 fn demo_grammar() -> Grammar {
725 checked_grammar_from_schema(&demo_schema(), &IdGen::new())
726 .expect("demo schema generates a clean grammar")
727 }
728
729 /// Parse + conformance-check against the demo schema in one go.
730 fn parse_ok(input: &str) -> crate::ParseTree {
731 let g = demo_grammar();
732 let tree = g
733 .parse(input)
734 .unwrap_or_else(|e| panic!("parse failed for {input:?}: {:?}", e.diagnostics));
735 let diags = check_conformance(&tree, &demo_schema());
736 assert!(
737 diags.is_empty(),
738 "conformance clean for {input:?}: {diags:?}"
739 );
740 tree
741 }
742
743 #[test]
744 fn generated_grammar_passes_all_static_checks() {
745 // checked_grammar_from_schema already runs the three checks;
746 // this test pins that it succeeds for the full shape matrix.
747 demo_grammar();
748 }
749
750 #[test]
751 fn parses_int_field_variant() {
752 let tree = parse_ok("Lit(value: -42)");
753 assert_eq!(tree.variant, "Lit");
754 assert_eq!(tree.field("value"), Some(&RawValue::Text("-42".into())));
755 }
756
757 #[test]
758 fn parses_string_and_bool_fields_with_escapes() {
759 let tree = parse_ok(r#"Name(text: "hello \"world\"\n", quoted: true)"#);
760 assert_eq!(
761 tree.field("text"),
762 Some(&RawValue::Text("hello \"world\"\n".into()))
763 );
764 assert_eq!(tree.field("quoted"), Some(&RawValue::Text("true".into())));
765 }
766
767 #[test]
768 fn parses_nested_one_children() {
769 let tree = parse_ok("Add(lhs: Lit(value: 1), rhs: Neg(body: none))");
770 let lhs = tree.child_slot("lhs").unwrap();
771 assert_eq!(lhs.len(), 1);
772 assert_eq!(lhs[0].variant, "Lit");
773 let rhs = tree.child_slot("rhs").unwrap();
774 assert_eq!(rhs[0].variant, "Neg");
775 }
776
777 #[test]
778 fn optional_child_none_leaves_slot_absent() {
779 let tree = parse_ok("Neg(body: none)");
780 assert!(
781 tree.child_slot("body").is_none(),
782 "bare `none` = absent slot"
783 );
784 }
785
786 #[test]
787 fn optional_child_present_binds_one_tree() {
788 let tree = parse_ok("Neg(body: Lit(value: 3))");
789 assert_eq!(tree.child_slot("body").unwrap().len(), 1);
790 }
791
792 #[test]
793 fn many_child_list_binds_in_order_and_drops_commas() {
794 let tree = parse_ok("List(items: [Lit(value: 1), Lit(value: 2), Unit()])");
795 let items = tree.child_slot("items").unwrap();
796 let variants: Vec<&str> = items.iter().map(|t| t.variant.as_str()).collect();
797 assert_eq!(variants, ["Lit", "Lit", "Unit"]);
798 }
799
800 #[test]
801 fn many_child_empty_list_is_conformant() {
802 let tree = parse_ok("List(items: [])");
803 assert!(
804 tree.child_slot("items").is_none_or(<[_]>::is_empty),
805 "empty list = zero children"
806 );
807 }
808
809 #[test]
810 fn keyed_map_binds_entries_sorted_by_key() {
811 // Written deliberately out of order, with one quoted key that
812 // could not be spelled as a bare identifier: the parser is
813 // responsible for landing both in canonical order.
814 let tree = parse_ok(r#"Env(entries: { zeta: Unit(), "a b": Lit(value: 1) })"#);
815 let entries = tree.keyed_child_slot("entries").expect("keyed slot bound");
816 let keys: Vec<&str> = entries.iter().map(|(k, _)| k.as_str()).collect();
817 assert_eq!(keys, ["a b", "zeta"]);
818 let variants: Vec<&str> = entries.iter().map(|(_, t)| t.variant.as_str()).collect();
819 assert_eq!(variants, ["Lit", "Unit"]);
820 assert!(
821 tree.children.is_empty(),
822 "keyed entries must not leak into the positional half: {:?}",
823 tree.children
824 );
825 }
826
827 #[test]
828 fn keyed_map_source_order_does_not_change_the_tree() {
829 let forward = parse_ok("Env(entries: { a: Unit(), b: Lit(value: 1) })");
830 let reversed = parse_ok("Env(entries: { b: Lit(value: 1), a: Unit() })");
831 // Spans differ (the two spellings put the subtrees at
832 // different offsets), so compare the keyed payload rather than
833 // the whole tree.
834 let keys = |t: &crate::ParseTree| -> Vec<String> {
835 t.keyed_child_slot("entries")
836 .unwrap()
837 .iter()
838 .map(|(k, v)| format!("{k}={}", v.variant))
839 .collect()
840 };
841 assert_eq!(keys(&forward), keys(&reversed));
842 }
843
844 #[test]
845 fn keyed_map_empty_is_conformant() {
846 let tree = parse_ok("Env(entries: {})");
847 assert!(
848 tree.keyed_child_slot("entries").is_none_or(<[_]>::is_empty),
849 "empty map = zero entries"
850 );
851 }
852
853 #[test]
854 fn keyed_map_duplicate_key_parses_but_fails_conformance() {
855 // The grammar has no reason to reject it — duplicate detection
856 // is the schema's job, and `parse_ok` would hide the split, so
857 // this test drives the two stages apart deliberately.
858 let g = demo_grammar();
859 let tree = g
860 .parse("Env(entries: { k: Unit(), k: Lit(value: 1) })")
861 .expect("grammar accepts a repeated key");
862 let diags = check_conformance(&tree, &demo_schema());
863 assert!(
864 diags.iter().any(|d| d.code == crate::codes::DUPLICATE_KEY),
865 "expected DUPLICATE_KEY; got {:?}",
866 diags.iter().map(|d| d.code.as_str()).collect::<Vec<_>>()
867 );
868 }
869
870 #[test]
871 fn zero_argument_variant_parses() {
872 let tree = parse_ok("Unit()");
873 assert_eq!(tree.variant, "Unit");
874 assert!(tree.fields.is_empty() && tree.children.is_empty());
875 }
876
877 #[test]
878 fn whitespace_and_newlines_are_tolerated() {
879 let tree = parse_ok("Add(\n lhs: Lit( value : 7 ),\n rhs: Unit()\n)");
880 assert_eq!(tree.child_slot("lhs").unwrap()[0].variant, "Lit");
881 }
882
883 #[test]
884 fn unsupported_field_type_fails_generation_with_full_list() {
885 let schema = NodeSchema {
886 name: "Bad".into(),
887 variants: vec![VariantSchema {
888 name: "Par".into(),
889 fields: vec![
890 FieldSchema {
891 name: "policy".into(),
892 ty: "Option<JoinPolicy>".into(),
893 optional: false,
894 },
895 FieldSchema {
896 name: "reducer".into(),
897 ty: "ReducerId".into(),
898 optional: false,
899 },
900 ],
901 children: vec![],
902 }],
903 };
904 let err = grammar_from_schema(&schema, &IdGen::new()).unwrap_err();
905 assert_eq!(err.diagnostics.len(), 2, "one diagnostic per bad field");
906 assert!(
907 err.diagnostics
908 .iter()
909 .all(|d| d.code == codes::UNSUPPORTED_FIELD)
910 );
911 }
912
913 /// A scalar-valued keyed slot is recognised by the derive and
914 /// the JSON front-end, but the canonical text syntax has no
915 /// production for it yet — `grammar_from_schema` must fail
916 /// loudly with [`codes::UNSUPPORTED_MAP_VALUE_SHAPE`] rather
917 /// than silently emit a grammar that expects each entry's value
918 /// to be a full AST node. Pins the pre-flight guard so a
919 /// downstream feeding a Shape 1 schema through PEG generation
920 /// gets a clear pointer to the JSON front-end instead of an
921 /// opaque parse failure.
922 #[test]
923 fn scalar_keyed_slot_fails_grammar_generation_loudly() {
924 let schema = NodeSchema {
925 name: "Cfg".into(),
926 variants: vec![VariantSchema {
927 name: "Env".into(),
928 fields: vec![],
929 children: vec![ChildSchema::scalar_map("entries", "String")],
930 }],
931 };
932 let err = grammar_from_schema(&schema, &IdGen::new()).unwrap_err();
933 assert_eq!(err.diagnostics.len(), 1);
934 assert_eq!(err.diagnostics[0].code, codes::UNSUPPORTED_MAP_VALUE_SHAPE);
935 assert!(
936 err.diagnostics[0].message.contains("entries"),
937 "diagnostic mentions the slot name: {}",
938 err.diagnostics[0].message
939 );
940 assert!(
941 err.diagnostics[0].message.contains("String"),
942 "diagnostic mentions the scalar value type: {}",
943 err.diagnostics[0].message
944 );
945 }
946
947 /// Recursive keyed slots still generate cleanly — the scalar
948 /// gate must not accidentally reject the pre-0.6 keyed shape.
949 #[test]
950 fn recursive_keyed_slot_still_generates_grammar() {
951 let schema = NodeSchema {
952 name: "Cfg".into(),
953 variants: vec![
954 VariantSchema {
955 name: "Env".into(),
956 fields: vec![],
957 children: vec![ChildSchema::recursive("entries", Multiplicity::Map)],
958 },
959 VariantSchema {
960 name: "Unit".into(),
961 fields: vec![],
962 children: vec![],
963 },
964 ],
965 };
966 grammar_from_schema(&schema, &IdGen::new())
967 .expect("recursive keyed shape still supported end-to-end in the text front-end");
968 }
969
970 /// Flow-shaped schema whose `Par` payload fields the built-in
971 /// mapping rejects — the override motivating case.
972 fn par_schema() -> NodeSchema {
973 NodeSchema {
974 name: "Flow".into(),
975 variants: vec![
976 VariantSchema {
977 name: "Par".into(),
978 fields: vec![
979 FieldSchema {
980 name: "policy".into(),
981 // Token-stream spelling, as the derive extracts it.
982 ty: "Option < JoinPolicy >".into(),
983 optional: false,
984 },
985 FieldSchema {
986 name: "reducer_id".into(),
987 ty: "Option < String >".into(),
988 optional: false,
989 },
990 ],
991 children: vec![ChildSchema {
992 name: "children".into(),
993 multiplicity: Multiplicity::Many,
994 value_shape: ChildValueShape::Recursive,
995 }],
996 },
997 VariantSchema {
998 name: "Call".into(),
999 fields: vec![FieldSchema {
1000 name: "label".into(),
1001 ty: "String".into(),
1002 optional: false,
1003 }],
1004 children: vec![],
1005 },
1006 ],
1007 }
1008 }
1009
1010 fn par_overrides() -> SyntaxOverrides {
1011 SyntaxOverrides::new()
1012 .for_type("Option<JoinPolicy>", |ids| {
1013 choice(
1014 ids,
1015 vec![
1016 token(ids, "%kw:none"),
1017 token(ids, "%kw:all_failfast"),
1018 token(ids, "%kw:any_failfast"),
1019 ],
1020 )
1021 })
1022 .for_type("Option<String>", |ids| {
1023 choice(ids, vec![token(ids, "%kw:none"), token(ids, "%str")])
1024 })
1025 }
1026
1027 #[test]
1028 fn type_overrides_unlock_exotic_payload_fields() {
1029 // Without overrides the Flow-shaped schema fails generation
1030 // (pinned by unsupported_field_type_fails_generation_with_full_list);
1031 // with them it generates check-clean and parses. Note the
1032 // override keys are written without whitespace while the schema
1033 // carries the token-stream spelling.
1034 let g = checked_grammar_from_schema_with(&par_schema(), &IdGen::new(), &par_overrides())
1035 .expect("overridden Flow schema generates a clean grammar");
1036 let tree = g
1037 .parse(
1038 r#"Par(policy: all_failfast, reducer_id: "reduce_all_ordered",
1039 children: [Call(label: "a"), Call(label: "b")])"#,
1040 )
1041 .expect("canonical Par text parses");
1042 let diags = check_conformance(&tree, &par_schema());
1043 assert!(diags.is_empty(), "conformance clean: {diags:?}");
1044 assert_eq!(
1045 tree.field("policy"),
1046 Some(&RawValue::Text("all_failfast".into()))
1047 );
1048 assert_eq!(
1049 tree.field("reducer_id"),
1050 Some(&RawValue::Text("reduce_all_ordered".into()))
1051 );
1052 assert_eq!(tree.child_slot("children").unwrap().len(), 2);
1053 }
1054
1055 #[test]
1056 fn field_override_wins_over_type_override() {
1057 // Same type in both entries; the (variant, field)-pinned one
1058 // must decide the syntax.
1059 let overrides = par_overrides().for_field("Par", "policy", |ids| token(ids, "%kw:pinned"));
1060 let g = checked_grammar_from_schema_with(&par_schema(), &IdGen::new(), &overrides)
1061 .expect("clean grammar with field-level override");
1062 let tree = g
1063 .parse(r#"Par(policy: pinned, reducer_id: none, children: [])"#)
1064 .expect("field-level syntax parses");
1065 assert_eq!(tree.field("policy"), Some(&RawValue::Text("pinned".into())));
1066 // The type-level spelling is no longer accepted at that site.
1067 assert!(
1068 g.parse(r#"Par(policy: all_failfast, reducer_id: none, children: [])"#)
1069 .is_err(),
1070 "type-level syntax rejected once the field is pinned"
1071 );
1072 }
1073
1074 #[test]
1075 fn unrelated_override_leaves_builtin_mapping_and_failure_intact() {
1076 // An override for some other type neither rescues the
1077 // unsupported fields nor perturbs built-in ones. Note that
1078 // `Option<String>` on `reducer_id` is now a built-in mapping
1079 // (Layer 2 of the "built-in optional payload fields"
1080 // contract), so only the `Option<JoinPolicy>` field on
1081 // `policy` remains unmapped without a `SyntaxOverrides` entry.
1082 let overrides = SyntaxOverrides::new().for_type("Uuid", |ids| token(ids, "%str"));
1083 let err = grammar_from_schema_with(&par_schema(), &IdGen::new(), &overrides).unwrap_err();
1084 assert_eq!(
1085 err.diagnostics.len(),
1086 1,
1087 "policy is the only Par field still unmapped without overrides",
1088 );
1089 assert!(
1090 err.diagnostics
1091 .iter()
1092 .all(|d| d.code == codes::UNSUPPORTED_FIELD)
1093 );
1094 assert!(
1095 err.diagnostics[0].message.contains("policy"),
1096 "unmapped-field diagnostic points at `policy`",
1097 );
1098 // Built-in demo schema is unaffected by the stray entry.
1099 let g = checked_grammar_from_schema_with(&demo_schema(), &IdGen::new(), &overrides)
1100 .expect("demo schema still generates");
1101 assert_eq!(g.parse("Lit(value: 7)").unwrap().variant, "Lit");
1102 }
1103
1104 /// Layer 2: `Option<String>` payload maps to `none | %str` via the
1105 /// built-in mapping (no `SyntaxOverrides` entry needed). Canonical
1106 /// text parses both spellings and both build through the derive's
1107 /// `build_field_optional` route.
1108 #[test]
1109 fn builtin_option_string_mapping_accepts_none_and_str() {
1110 let schema = NodeSchema {
1111 name: "Meta".into(),
1112 variants: vec![VariantSchema {
1113 name: "Row".into(),
1114 fields: vec![FieldSchema {
1115 name: "description".into(),
1116 ty: "Option<String>".into(),
1117 optional: true,
1118 }],
1119 children: vec![],
1120 }],
1121 };
1122 let g = checked_grammar_from_schema(&schema, &IdGen::new())
1123 .expect("Option<String> is a built-in mapping");
1124 // `none` spelling.
1125 let t1 = g.parse(r#"Row(description: none)"#).expect("none parses");
1126 assert_eq!(
1127 t1.field("description"),
1128 Some(&RawValue::Text("none".into())),
1129 );
1130 // `%str` spelling.
1131 let t2 = g.parse(r#"Row(description: "hi")"#).expect("string parses");
1132 assert_eq!(t2.field("description"), Some(&RawValue::Text("hi".into())));
1133 // Omitted entirely (Layer 1 optional PEG wrap).
1134 let t3 = g.parse(r#"Row()"#).expect("omitted parses");
1135 assert!(t3.field("description").is_none());
1136 // check_conformance is clean for all three.
1137 for t in [&t1, &t2, &t3] {
1138 assert!(
1139 check_conformance(t, &schema).is_empty(),
1140 "conformance clean for {t:?}",
1141 );
1142 }
1143 }
1144
1145 /// Layer 2: `Vec<String>` payload maps to `[ %str ("," %str)* ]`
1146 /// via the built-in mapping. Empty list, single item, and multi
1147 /// item all parse; captured text is JSON-parsable so
1148 /// `build_field_vec` reads it directly.
1149 #[test]
1150 fn builtin_vec_string_mapping_accepts_bracketed_list() {
1151 let schema = NodeSchema {
1152 name: "Meta".into(),
1153 variants: vec![VariantSchema {
1154 name: "Row".into(),
1155 fields: vec![FieldSchema {
1156 name: "tags".into(),
1157 ty: "Vec<String>".into(),
1158 optional: true,
1159 }],
1160 children: vec![],
1161 }],
1162 };
1163 let g = checked_grammar_from_schema(&schema, &IdGen::new())
1164 .expect("Vec<String> is a built-in mapping");
1165 // Empty list.
1166 let t1 = g.parse(r#"Row(tags: [])"#).expect("empty list parses");
1167 assert_eq!(t1.field("tags"), Some(&RawValue::Text("[]".into())));
1168 // Multi item. The PEG runtime's implicit whitespace-skip
1169 // strips inter-token spaces from the captured text, so the
1170 // field carries the compact form `["a","b"]` (still valid
1171 // JSON — `serde_json::from_str` accepts it).
1172 let t2 = g
1173 .parse(r#"Row(tags: ["a", "b"])"#)
1174 .expect("two items parses");
1175 assert_eq!(
1176 t2.field("tags"),
1177 Some(&RawValue::Text(r#"["a","b"]"#.into())),
1178 );
1179 // Omitted.
1180 let t3 = g.parse(r#"Row()"#).expect("omitted parses");
1181 assert!(t3.field("tags").is_none());
1182 for t in [&t1, &t2, &t3] {
1183 assert!(
1184 check_conformance(t, &schema).is_empty(),
1185 "conformance clean for {t:?}",
1186 );
1187 }
1188 }
1189
1190 #[test]
1191 fn empty_schema_fails_generation() {
1192 let schema = NodeSchema {
1193 name: "Void".into(),
1194 variants: vec![],
1195 };
1196 let err = grammar_from_schema(&schema, &IdGen::new()).unwrap_err();
1197 assert_eq!(err.diagnostics[0].code, codes::EMPTY_SCHEMA);
1198 }
1199
1200 #[test]
1201 fn parse_error_reports_expected_set() {
1202 let g = demo_grammar();
1203 let err = g.parse("Lit(value: banana)").unwrap_err();
1204 assert!(!err.diagnostics.is_empty());
1205 }
1206}