dsl-kit-parse 0.4.0

Parser trunk for dsl-kit DSLs (ParseTree, schema conformance, serde bridge, DslBuild consumer).
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
//! Example synthesis from a grammar (item Q-1).
//!
//! Walks a [`Grammar`]'s [`Peg`] — not the schema — and emits input
//! strings that the grammar accepts **by construction**: every choice
//! picks a concrete arm, every token contributes matching input text
//! (`%int` → `1`, `%str` → `"example"`, keywords verbatim). Because
//! synthesis reads the grammar itself, spellings supplied through
//! `schema_gen::SyntaxOverrides` fall out automatically — no separate
//! example registration.
//!
//! Two synthesis modes drive the output of [`examples_from_grammar`]:
//!
//! - **Minimal** (per-rule examples) — at every choice take the arm
//!   with the cheapest finite derivation, at every repeat emit the
//!   minimum count. Terminates by the shortest-derivation argument:
//!   each step either emits a token or strictly reduces the remaining
//!   derivation cost.
//! - **Rich** (the composite program) — while a depth budget lasts,
//!   choices take the *most expensive* finite arm and repeats emit up
//!   to two iterations, showing nesting and list syntax; the budget
//!   decrements per rule expansion and exhaustion falls back to
//!   Minimal.
//!
//! The intended consumer is an AI writing the canonical text syntax:
//! few-shot examples beat a formal grammar table for one-shot
//! accuracy, and machine-derived examples cannot drift from the
//! grammar the way hand-written documentation does.

use std::collections::HashMap;

use crate::peg::{Grammar, Peg};
use crate::{BuildError, Diagnostic};

/// Diagnostic codes emitted by example synthesis.
pub mod codes {
    /// A rule has no finite derivation (every path recurses forever),
    /// so no example input exists.
    pub const NO_FINITE_DERIVATION: &str = "dsl_kit::example_gen::no_finite_derivation";
    /// A `RuleRef` names a rule the grammar does not define.
    pub const UNKNOWN_RULE: &str = "dsl_kit::example_gen::unknown_rule";
}

/// Rich-mode depth budget used by [`examples_from_grammar`] for the
/// composite program: rule expansions beyond this depth continue
/// minimally.
pub const COMPOSITE_DEPTH: u32 = 2;

/// One rule's minimal example.
#[derive(Debug, Clone)]
pub struct RuleExample {
    /// Rule name (for `schema_gen` grammars: the variant name).
    pub rule: String,
    /// Input text accepted by the grammar, entering at this rule.
    pub text: String,
}

/// Everything [`examples_from_grammar`] synthesizes for one grammar.
#[derive(Debug, Clone)]
pub struct GrammarExamples {
    /// Minimal example per non-start rule, in rule declaration order.
    /// For `schema_gen` grammars this is exactly one example per
    /// variant.
    pub per_rule: Vec<RuleExample>,
    /// One depth-capped rich expansion of the start rule, showing
    /// nesting and repetition.
    pub composite: String,
}

/// Synthesizes [`GrammarExamples`] for `grammar`.
///
/// Fails with one [`codes::NO_FINITE_DERIVATION`] diagnostic per
/// underivable rule (collected, so the author sees the full list), or
/// [`codes::UNKNOWN_RULE`] on a dangling reference.
pub fn examples_from_grammar(grammar: &Grammar) -> Result<GrammarExamples, BuildError> {
    let synth = Synth::prepare(grammar)?;

    let mut underivable = Vec::new();
    for (name, _) in &synth.rules_in_order {
        if synth.rule_cost(name) == INFINITE {
            underivable.push(Diagnostic::error(
                codes::NO_FINITE_DERIVATION,
                format!("rule `{name}` has no finite derivation — no example input exists"),
            ));
        }
    }
    if !underivable.is_empty() {
        return Err(BuildError::new(underivable));
    }

    let mut per_rule = Vec::new();
    for (name, body) in &synth.rules_in_order {
        if *name == grammar.start {
            continue;
        }
        let mut tokens = Vec::new();
        synth.emit(body, 0, &mut tokens)?;
        per_rule.push(RuleExample {
            rule: name.clone(),
            text: render(&tokens),
        });
    }

    let start_body = synth.rule_body(&grammar.start)?;
    let mut tokens = Vec::new();
    synth.emit(start_body, COMPOSITE_DEPTH, &mut tokens)?;
    let composite = render(&tokens);

    Ok(GrammarExamples {
        per_rule,
        composite,
    })
}

/// Sentinel cost for "no finite derivation".
const INFINITE: u64 = u64::MAX;

struct Synth<'g> {
    /// Rule name → body, for `RuleRef` expansion.
    by_name: HashMap<&'g str, &'g Peg>,
    /// `(name, body)` in grammar declaration order.
    rules_in_order: Vec<(String, &'g Peg)>,
    /// Shortest-derivation token count per rule (fixpoint result).
    costs: HashMap<String, u64>,
}

impl<'g> Synth<'g> {
    fn prepare(grammar: &'g Grammar) -> Result<Self, BuildError> {
        let mut by_name = HashMap::new();
        let mut rules_in_order = Vec::new();
        for r in &grammar.rules {
            if let Peg::Rule { name, body, .. } = r {
                by_name.insert(name.as_str(), body.as_ref());
                rules_in_order.push((name.clone(), body.as_ref()));
            }
        }

        // Shortest-derivation cost fixpoint: start every rule at
        // INFINITE and relax until stable. Monotone non-increasing and
        // bounded, so it terminates.
        let mut costs: HashMap<String, u64> = rules_in_order
            .iter()
            .map(|(n, _)| (n.clone(), INFINITE))
            .collect();
        loop {
            let mut changed = false;
            for (name, body) in &rules_in_order {
                let c = peg_cost(body, &costs);
                if c < costs[name] {
                    costs.insert(name.clone(), c);
                    changed = true;
                }
            }
            if !changed {
                break;
            }
        }

        Ok(Self {
            by_name,
            rules_in_order,
            costs,
        })
    }

    fn rule_cost(&self, name: &str) -> u64 {
        self.costs.get(name).copied().unwrap_or(INFINITE)
    }

    fn rule_body(&self, name: &str) -> Result<&'g Peg, BuildError> {
        self.by_name.get(name).copied().ok_or_else(|| {
            BuildError::single(Diagnostic::error(
                codes::UNKNOWN_RULE,
                format!("reference to undefined rule `{name}`"),
            ))
        })
    }

    /// Appends the input tokens of one derivation of `peg`.
    /// `rich_depth > 0` selects the expansive arm at choices and up to
    /// two iterations at repeats; `0` is the minimal mode.
    fn emit(&self, peg: &Peg, rich_depth: u32, out: &mut Vec<String>) -> Result<(), BuildError> {
        match peg {
            Peg::Token { pat, .. } => {
                out.push(token_input(pat));
                Ok(())
            }
            Peg::Seq { items, .. } => {
                for item in items {
                    self.emit(item, rich_depth, out)?;
                }
                Ok(())
            }
            Peg::Choice { alts, .. } => {
                let arm = self.pick_arm(alts, rich_depth)?;
                self.emit(arm, rich_depth, out)
            }
            Peg::Repeat { body, min, max, .. } => {
                let count = if rich_depth > 0 {
                    (*min).max(2.min(max.unwrap_or(2)))
                } else {
                    *min
                };
                for _ in 0..count {
                    self.emit(body, rich_depth, out)?;
                }
                Ok(())
            }
            Peg::RuleRef { name, .. } => {
                let body = self.rule_body(name)?;
                self.emit(body, rich_depth.saturating_sub(1), out)
            }
            Peg::Rule { body, .. } | Peg::Node { body, .. } | Peg::Field { body, .. } => {
                self.emit(body, rich_depth, out)
            }
        }
    }

    /// Chooses a choice arm: cheapest finite derivation in minimal
    /// mode, most expensive finite one in rich mode (ties: first).
    fn pick_arm<'p>(&self, alts: &'p [Peg], rich_depth: u32) -> Result<&'p Peg, BuildError> {
        let finite = alts
            .iter()
            .map(|a| (a, peg_cost(a, &self.costs)))
            .filter(|(_, c)| *c != INFINITE);
        let picked = if rich_depth > 0 {
            finite.max_by_key(|(_, c)| *c)
        } else {
            finite.min_by_key(|(_, c)| *c)
        };
        picked.map(|(a, _)| a).ok_or_else(|| {
            BuildError::single(Diagnostic::error(
                codes::NO_FINITE_DERIVATION,
                "choice has no finite alternative".to_string(),
            ))
        })
    }
}

/// Shortest-derivation token count of `peg` under the current rule
/// cost estimates ([`INFINITE`]-safe saturating arithmetic).
fn peg_cost(peg: &Peg, costs: &HashMap<String, u64>) -> u64 {
    match peg {
        Peg::Token { .. } => 1,
        Peg::Seq { items, .. } => items
            .iter()
            .try_fold(0u64, |acc, i| {
                let c = peg_cost(i, costs);
                if c == INFINITE {
                    None
                } else {
                    Some(acc.saturating_add(c))
                }
            })
            .unwrap_or(INFINITE),
        Peg::Choice { alts, .. } => alts
            .iter()
            .map(|a| peg_cost(a, costs))
            .min()
            .unwrap_or(INFINITE),
        Peg::Repeat { body, min, .. } => {
            if *min == 0 {
                0
            } else {
                let c = peg_cost(body, costs);
                if c == INFINITE {
                    INFINITE
                } else {
                    c.saturating_mul(u64::from(*min))
                }
            }
        }
        Peg::RuleRef { name, .. } => costs.get(name).copied().unwrap_or(INFINITE),
        Peg::Rule { body, .. } | Peg::Node { body, .. } | Peg::Field { body, .. } => {
            peg_cost(body, costs)
        }
    }
}

/// Input text matching one token pattern.
fn token_input(pat: &str) -> String {
    if let Some(kw) = pat.strip_prefix("%kw:") {
        return kw.to_string();
    }
    match pat {
        "%int" => "1".to_string(),
        "%str" => "\"example\"".to_string(),
        "%ident" => "x".to_string(),
        "%ws" => " ".to_string(),
        literal => literal.to_string(),
    }
}

/// Joins tokens with canonical-syntax spacing: no space before
/// punctuation that closes or separates (`,` `)` `]` `:` `(`), no
/// space after an opener (`(` `[`).
fn render(tokens: &[String]) -> String {
    let mut out = String::new();
    for t in tokens {
        let tight_before = matches!(t.as_str(), "," | ")" | "]" | ":" | "(");
        let tight_after_prev = out.ends_with('(') || out.ends_with('[');
        if !out.is_empty() && !tight_before && !tight_after_prev {
            out.push(' ');
        }
        out.push_str(t);
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::check_conformance;
    use crate::schema_gen::checked_grammar_from_schema;
    use dsl_kit_core::IdGen;
    use dsl_kit_schema::{ChildSchema, FieldSchema, Multiplicity, NodeSchema, VariantSchema};

    /// Same shape matrix as the schema_gen demo: int / string / bool
    /// fields, One / Optional / Many children, zero-argument variant.
    fn demo_schema() -> NodeSchema {
        NodeSchema {
            name: "Expr".into(),
            variants: vec![
                VariantSchema {
                    name: "Lit".into(),
                    fields: vec![FieldSchema {
                        name: "value".into(),
                        ty: "i64".into(),
                        optional: false,
                    }],
                    children: vec![],
                },
                VariantSchema {
                    name: "Name".into(),
                    fields: vec![
                        FieldSchema {
                            name: "text".into(),
                            ty: "String".into(),
                            optional: false,
                        },
                        FieldSchema {
                            name: "quoted".into(),
                            ty: "bool".into(),
                            optional: false,
                        },
                    ],
                    children: vec![],
                },
                VariantSchema {
                    name: "Add".into(),
                    fields: vec![],
                    children: vec![
                        ChildSchema {
                            name: "lhs".into(),
                            multiplicity: Multiplicity::One,
                        },
                        ChildSchema {
                            name: "rhs".into(),
                            multiplicity: Multiplicity::One,
                        },
                    ],
                },
                VariantSchema {
                    name: "Neg".into(),
                    fields: vec![],
                    children: vec![ChildSchema {
                        name: "body".into(),
                        multiplicity: Multiplicity::Optional,
                    }],
                },
                VariantSchema {
                    name: "List".into(),
                    fields: vec![],
                    children: vec![ChildSchema {
                        name: "items".into(),
                        multiplicity: Multiplicity::Many,
                    }],
                },
                VariantSchema {
                    name: "Unit".into(),
                    fields: vec![],
                    children: vec![],
                },
            ],
        }
    }

    fn demo_examples() -> (Grammar, GrammarExamples) {
        let g = checked_grammar_from_schema(&demo_schema(), &IdGen::new())
            .expect("demo schema generates");
        let ex = examples_from_grammar(&g).expect("examples synthesize");
        (g, ex)
    }

    #[test]
    fn every_per_rule_example_parses_as_its_own_variant() {
        let (g, ex) = demo_examples();
        assert_eq!(ex.per_rule.len(), 6, "one example per variant");
        for e in &ex.per_rule {
            let tree = g.parse(&e.text).unwrap_or_else(|err| {
                panic!(
                    "example for `{}` failed to parse: {:?}\n  text: {}",
                    e.rule, err.diagnostics, e.text
                )
            });
            assert_eq!(tree.variant, e.rule, "example enters at its own variant");
            let diags = check_conformance(&tree, &demo_schema());
            assert!(
                diags.is_empty(),
                "conformance clean for {}: {diags:?}",
                e.rule
            );
        }
    }

    #[test]
    fn minimal_examples_have_the_expected_spellings() {
        let (_, ex) = demo_examples();
        let by_rule: HashMap<&str, &str> = ex
            .per_rule
            .iter()
            .map(|e| (e.rule.as_str(), e.text.as_str()))
            .collect();
        assert_eq!(by_rule["Lit"], "Lit(value: 1)");
        assert_eq!(by_rule["Name"], "Name(text: \"example\", quoted: true)");
        assert_eq!(
            by_rule["Add"], "Add(lhs: Unit(), rhs: Unit())",
            "nested slots fill with the cheapest variant"
        );
        assert_eq!(
            by_rule["Neg"], "Neg(body: none)",
            "Optional minimal = absent"
        );
        assert_eq!(
            by_rule["List"], "List(items: [])",
            "Many minimal = empty list"
        );
        assert_eq!(by_rule["Unit"], "Unit()");
    }

    #[test]
    fn composite_parses_and_shows_nesting() {
        let (g, ex) = demo_examples();
        let tree = g.parse(&ex.composite).unwrap_or_else(|err| {
            panic!(
                "composite failed to parse: {:?}\n  text: {}",
                err.diagnostics, ex.composite
            )
        });
        assert!(check_conformance(&tree, &demo_schema()).is_empty());
        // Rich mode picked an expansive variant and expanded at least
        // one nested child — pin the property, not the exact string.
        assert!(
            !tree.children.is_empty(),
            "composite has child structure: {}",
            ex.composite
        );
    }

    #[test]
    fn underivable_rules_fail_loudly_with_the_full_list() {
        // A single variant whose only child is mandatory and recursive:
        // no finite tree exists.
        let schema = NodeSchema {
            name: "Loop".into(),
            variants: vec![VariantSchema {
                name: "Rec".into(),
                fields: vec![],
                children: vec![ChildSchema {
                    name: "inner".into(),
                    multiplicity: Multiplicity::One,
                }],
            }],
        };
        let g = crate::schema_gen::grammar_from_schema(&schema, &IdGen::new())
            .expect("generation itself succeeds");
        let err = examples_from_grammar(&g).expect_err("no finite derivation");
        assert!(
            err.diagnostics
                .iter()
                .all(|d| d.code == codes::NO_FINITE_DERIVATION),
            "{:?}",
            err.diagnostics
        );
        // Both the start rule and the variant rule are underivable.
        assert_eq!(err.diagnostics.len(), 2);
    }
}