badness-parser 0.12.0

Lossless CST parser, semantic model, and command-signature database for LaTeX and BibTeX
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
//! Math bodies, scripts, and paired `\left`/`\right` delimiters.
//!
//! All math bodies share atom and script parsing, including named environments.
//! Entry gates live with the grammar's shared shape-gate machinery.

use super::trivia::CommentMode;
use super::{Block, LEFT_CMD, Parser, RIGHT_CMD, peek_end_name};
use crate::parser::events::Event;
use crate::semantic::signature::ArgumentDomain;
use crate::syntax::SyntaxKind;

impl Parser<'_> {
    /// Inline `$ … $` or display `$$ … $$` math. The body's atoms are wrapped in
    /// a `MATH` node (the delimiters stay direct children of the math node); the
    /// atoms themselves are parsed in math mode (see [`Self::math_element`]).
    /// Entry is gated by [`Self::dollar_closes`]: the caller has already
    /// verified a closer is reachable, so the unclosed-math recovery paths
    /// below fire only for shapes the gate scan cannot see (they remain as
    /// belt-and-braces recovery, never the expected path).
    pub(super) fn dollar_math(&mut self) {
        let display = self.nth_kind(1) == Some(SyntaxKind::DOLLAR);
        let (kind, label) = if display {
            (SyntaxKind::DISPLAY_MATH, "$$")
        } else {
            (SyntaxKind::INLINE_MATH, "$")
        };
        let opener = (
            self.starts[self.pos],
            self.starts[self.pos + if display { 2 } else { 1 }],
        );
        let math = self.open(kind);
        self.bump(); // $
        if display {
            self.bump(); // second $
        }
        let body = self.open(SyntaxKind::MATH);
        self.math_dollar.push(true);
        loop {
            match self.kind() {
                None => {
                    self.error_at(opener, format!("unclosed `{label}`"));
                    break;
                }
                // `}` and `\end` are recovery anchors: `$`-math cannot span a
                // group or environment boundary, so a `}` here closes the
                // enclosing group (a math subgroup would have entered via `{`)
                // and a `\end` belongs to an enclosing environment. Leave the
                // token for the caller and report the unclosed math.
                Some(SyntaxKind::R_BRACE) => {
                    self.error_at(opener, format!("unclosed `{label}`"));
                    break;
                }
                Some(SyntaxKind::CONTROL_WORD) if self.at_env_end() => {
                    self.error_at(opener, format!("unclosed `{label}`"));
                    break;
                }
                Some(SyntaxKind::DOLLAR) => {
                    if display && self.nth_kind(1) != Some(SyntaxKind::DOLLAR) {
                        // A lone `$` inside `$$`: malformed; emit and continue.
                        self.bump();
                        continue;
                    }
                    // The closing delimiter belongs to the math node, not its
                    // body: break and bump it after closing `MATH`.
                    break;
                }
                _ => {
                    if self.at_paragraph_break() {
                        // Faithful to TeX: a blank line is a `\par`, and `\par`
                        // in math mode is "Missing $ inserted" — even inside an
                        // alignment cell (#35). Name the cause so the opener
                        // span isn't read as a bogus report.
                        self.error_at(
                            opener,
                            format!("unclosed `{label}` (a blank line ends math)"),
                        );
                        break;
                    }
                    self.math_element();
                }
            }
        }
        self.math_dollar.pop();
        self.close(body);
        if self.kind() == Some(SyntaxKind::DOLLAR) {
            self.bump(); // closing $
            if display {
                self.bump(); // second closing $
            }
        }
        self.close(math);
    }

    /// Delimited math: `\[ … \]` (display) or `\( … \)` (inline). As with
    /// [`Self::dollar_math`], the body's atoms are wrapped in a `MATH` node and
    /// parsed in math mode.
    pub(super) fn delim_math(&mut self, kind: SyntaxKind, opener: &str, closer: &str) {
        let opener_span = self.token_span(self.pos);
        let math = self.open(kind);
        self.bump(); // \[ or \(
        let body = self.open(SyntaxKind::MATH);
        self.math_dollar.push(false);
        loop {
            match self.kind() {
                None => {
                    self.error_at(opener_span, format!("unclosed `{opener}`"));
                    break;
                }
                Some(SyntaxKind::CONTROL_SYMBOL) if self.text() == closer => {
                    // The closer belongs to the math node, not its body.
                    break;
                }
                // A `}` closes an enclosing group: it cannot belong to this
                // math (a subgroup would have entered via `{`). Leave it for
                // the caller and report the unclosed math.
                Some(SyntaxKind::R_BRACE) => {
                    self.error_at(opener_span, format!("unclosed `{opener}`"));
                    break;
                }
                Some(SyntaxKind::CONTROL_WORD) if self.at_env_end() => {
                    self.error_at(opener_span, format!("unclosed `{opener}`"));
                    break;
                }
                _ => {
                    if self.at_paragraph_break() {
                        // Same rationale as in `dollar_math`: `\par` ends math.
                        self.error_at(
                            opener_span,
                            format!("unclosed `{opener}` (a blank line ends math)"),
                        );
                        break;
                    }
                    self.math_element();
                }
            }
        }
        self.math_dollar.pop();
        self.close(body);
        if self.kind() == Some(SyntaxKind::CONTROL_SYMBOL) && self.text() == closer {
            self.bump(); // \] or \)
        }
        self.close(math);
    }

    /// One element inside a math body. Trivia is emitted inline (for
    /// losslessness); everything else is an atom, possibly carrying `^`/`_`
    /// scripts (see [`Self::math_scripted`]). Callers guard the math closers and
    /// recovery anchors before invoking this, so the cursor is at body content.
    pub(super) fn math_element(&mut self) {
        match self.kind() {
            Some(k) if Self::is_trivia(k) => self.bump(),
            _ => self.math_scripted(),
        }
    }

    /// A base atom with any tightly-bound `^`/`_` scripts — the one sanctioned
    /// Pratt site (`AGENTS.md`, decision #3). Sub/superscripts are postfix with a
    /// single-atom right operand, so this is a base atom followed by a postfix
    /// loop, not full precedence climbing.
    ///
    /// We only wrap the base in a `SCRIPTED` node when a script actually
    /// attaches, so an unscripted atom stays a bare token/node (matching the
    /// `LINE_BREAK`-only-when-modifiers idiom). Because the base atom's extent is
    /// not known until parsed (a command greedily attaches its args), we parse it
    /// first and, if a script follows, retroactively splice a `SCRIPTED` start
    /// event in front of it via the event layer's [`super::Marker::precede`].
    fn math_scripted(&mut self) {
        // The lexer keeps ordinary characters in coarse `WORD` runs. Preserve an
        // unscripted run as one CST token. When a script follows, expose only the
        // final Unicode scalar as TeX's one-token base without changing the lexer.
        if self.kind() == Some(SyntaxKind::WORD) {
            let idx = self.pos;
            self.pos += 1;
            if self.at_script() {
                let end = self.tokens[idx].text.len();
                self.math_word_fragment(idx, 0, end);
            } else {
                self.events.push(Event::Tok(idx));
            }
            return;
        }
        let checkpoint = self.events.len();
        self.math_atom();
        self.math_scripts(checkpoint);
    }

    /// Emit one unconsumed byte range of a lexer `WORD`. If a script follows, the
    /// final Unicode scalar is isolated as its base; TeX tokenizes an ordinary
    /// input character separately even though the lossless lexer coalesces such
    /// characters.
    ///
    /// A fragment can be the remainder of a bare script argument (`x^23_i` leaves
    /// `3` after the `2`). In that case `self.pos` already points beyond the lexer
    /// token, so a following script correctly binds to the fragment's final atom.
    fn math_word_fragment(&mut self, idx: usize, start: usize, end: usize) {
        debug_assert!(start < end, "math WORD fragment must be non-empty");
        if !self.at_script() {
            self.events.push(Event::SubTok { idx, start, end });
            return;
        }

        let last = self.tokens[idx].text[start..end]
            .char_indices()
            .next_back()
            .map(|(offset, _)| start + offset)
            .expect("a WORD fragment is non-empty");
        if start < last {
            self.events.push(Event::SubTok {
                idx,
                start,
                end: last,
            });
        }
        let checkpoint = self.events.len();
        self.events.push(Event::SubTok {
            idx,
            start: last,
            end,
        });
        self.math_scripts(checkpoint);
    }

    /// Attach any `^`/`_` scripts that follow the base atom emitted since
    /// `checkpoint`, retro-splicing a `SCRIPTED` wrapper in front of it
    /// ([`Self::precede`]). No script → the base stays a bare atom.
    fn math_scripts(&mut self, checkpoint: usize) {
        if !self.at_script() {
            return; // bare atom, no wrapper
        }
        let scripted = self.precede(checkpoint, SyntaxKind::SCRIPTED);
        let mut remainder = None;
        while self.at_script() {
            self.skip_trivia(); // trivia between base/scripts rides inside SCRIPTED
            let sub = self.kind() == Some(SyntaxKind::UNDERSCORE);
            let script = self.open(if sub {
                SyntaxKind::SUBSCRIPT
            } else {
                SyntaxKind::SUPERSCRIPT
            });
            self.bump(); // `_` or `^`
            remainder = self.math_script_arg();
            self.close(script);
            // The rest of a coalesced WORD is outer math content. Any next script
            // belongs to its final atom, not to the base we just closed.
            if remainder.is_some() {
                break;
            }
        }
        self.close(scripted);
        if let Some((idx, start, end)) = remainder {
            self.math_word_fragment(idx, start, end);
        }
    }

    /// True if a `^`/`_` script operator directly follows, skipping only
    /// `WHITESPACE`/`NEWLINE` (not a comment, which must end its line — so a
    /// script never binds across a comment) and not a blank line (a paragraph
    /// break ends the math).
    fn at_script(&self) -> bool {
        // `CommentMode::Stop`: a comment ends the line, so it stops the scan (and
        // is reported as the next meaningful token, which is not a script), rather
        // than being skipped as it is elsewhere. A blank line ends the math.
        let s = self.scan_trivia(self.pos, CommentMode::Stop);
        !s.saw_blank_line
            && matches!(
                s.next_kind,
                Some(SyntaxKind::CARET | SyntaxKind::UNDERSCORE)
            )
    }

    /// A single base atom: a `{…}` group (parsed in math mode), a command with
    /// its greedily-attached arguments, an environment, a `\\` line break, or one
    /// ordinary token. Always consumes at least one token.
    ///
    /// **Caller contract: the cursor must not be at EOF.** The `None` arm below
    /// consumes nothing and emits nothing, so a caller that reaches it from a
    /// loop spins until [`super::PARSER_STEP_LIMIT`] and panics far from the mistake.
    /// Every loop that reaches here guards EOF already — the four math bodies
    /// with an explicit `None` arm, `math_environment_body` through
    /// [`Self::at_block_end`], and [`Self::math_script_arg`] through its own
    /// missing-argument check — and this turns that unwritten contract into a
    /// tripwire that fires at the offending call instead.
    fn math_atom(&mut self) {
        debug_assert!(!self.at_end(), "math_atom at EOF: caller must guard first");
        match self.kind() {
            Some(SyntaxKind::L_BRACE) => self.math_group(),
            Some(SyntaxKind::CONTROL_WORD) => {
                // Same definition-body/expl3-region and brace-less gates as
                // [`Self::element`] (issues #45/#60).
                if !self.in_macro_code(self.pos) && self.at_env_begin() {
                    // The group-escape gate is mode-independent: braces remain
                    // TeX structure inside math, so an environment macro cannot
                    // consume the closing brace of the group that contains it.
                    if self.environment_escapes_group(self.pos) {
                        if let Some(name) = peek_end_name(self.tokens, self.pos) {
                            self.demoted_envs.insert(name.into_owned());
                        }
                        self.command();
                    } else {
                        self.environment();
                    }
                } else if !self.in_macro_code(self.pos) && self.at_env_end() {
                    // [`Self::at_block_end`] declines to end a math body at a
                    // `\end` that orphans a `\begin` the brace-group gate demoted
                    // (issue #71), so that one arrives *here* — and must land as
                    // the plain command that verdict already made it. Reporting
                    // it stray would have the two halves of one gate disagree.
                    if self.end_orphans_a_demoted_begin(self.pos) {
                        self.command();
                    } else {
                        self.stray_end();
                    }
                } else if let Some((target, closer)) = (!self.in_macro_code(self.pos))
                    .then(|| {
                        let target = self.alias_openers.get(&self.pos)?.clone();
                        Some((target, self.alias_closer(self.pos)?))
                    })
                    .flatten()
                {
                    // The [`Self::element`] arm, in math (issue #117). Not an
                    // optional extra: `split` — the environment the issue is
                    // about — is math-only, so an alias for it is *always* read
                    // here and nowhere else. `alias_environment` routes the body
                    // by the target exactly as `environment()` does one token
                    // earlier in this same match.
                    self.alias_environment(&target, closer);
                } else if self.at_command(LEFT_CMD) && self.left_right_closes(self.pos) {
                    self.left_right();
                } else if self.at_command(RIGHT_CMD) {
                    self.stray_right();
                } else {
                    self.command();
                }
            }
            // `\\` line break (with its tightly-bound `*`/`[len]`) vs. a bare
            // control symbol (`\,`, `\;`, `\!`, spacing) — emit the latter as a
            // single token.
            Some(SyntaxKind::CONTROL_SYMBOL) if self.text() == "\\\\" => self.line_break(),
            // Any other single token (WORD, digit, `&`, `~`, `#`, brackets, a
            // bare control symbol, or a `^`/`_` with no base): one token, so the
            // loop always makes progress.
            Some(_) => self.bump(),
            // Ruled out by the caller contract above; kept because release
            // builds compile the assert away and the match must be total.
            None => {}
        }
    }

    /// One script argument: a single atom (a `{…}` group, a command with its
    /// args, or one input character from a lexer `WORD`). The remainder of a
    /// `WORD` is returned to [`Self::math_scripts`] so it can become outer math
    /// content after the `SCRIPTED` node closes. A missing argument (the next
    /// meaningful token is a closer, `\end`, a paragraph break, or EOF) is
    /// reported, not consumed — the closer must stay for the enclosing math loop.
    fn math_script_arg(&mut self) -> Option<(usize, usize, usize)> {
        if self.at_paragraph_break() {
            self.error("missing argument after `^`/`_`");
            return None;
        }
        self.skip_trivia();
        let missing = match self.kind() {
            None | Some(SyntaxKind::R_BRACE | SyntaxKind::DOLLAR) => true,
            Some(SyntaxKind::CONTROL_SYMBOL) => matches!(self.text(), "\\]" | "\\)"),
            Some(SyntaxKind::CONTROL_WORD) => self.at_env_end(),
            _ => false,
        };
        if missing {
            self.error("missing argument after `^`/`_`");
            return None;
        }
        if self.kind() == Some(SyntaxKind::WORD) {
            let idx = self.pos;
            let text = &self.tokens[idx].text;
            let end = text.len();
            let first_end = text.char_indices().nth(1).map_or(end, |(offset, _)| offset);
            self.events.push(Event::SubTok {
                idx,
                start: 0,
                end: first_end,
            });
            self.pos += 1;
            return (first_end < end).then_some((idx, first_end, end));
        }
        self.math_atom();
        None
    }

    /// A brace group `{ … }` whose body is parsed in math mode (so `x^{a_b}`
    /// nests). Recovery mirrors [`Self::group`].
    fn math_group(&mut self) {
        self.argument_group(ArgumentDomain::Math);
    }

    /// A `\left<delim> … \right<delim>` matched delimiter pair (`AGENTS.md`,
    /// decision #3: the one precedence-climbing site — here just balanced
    /// matching by *count*, which is exactly how TeX pairs them, so a mismatched
    /// `\left( … \right]` still nests correctly). The `\left`/`\right` control
    /// words and their delimiter tokens are direct children (mirroring how `$` /
    /// `\[` delimiters stay direct children of the math node); the enclosed atoms
    /// are wrapped in a `MATH` body. Nested pairs recurse via [`Self::math_atom`].
    ///
    /// An unclosed `\left` recovers at the enclosing math/group/environment
    /// closer (the same anchors the surrounding math loop uses), leaving that
    /// token for the caller.
    fn left_right(&mut self) {
        debug_assert!(self.at_command(LEFT_CMD));
        let opener = self.token_span(self.pos);
        let pair = self.open(SyntaxKind::LEFT_RIGHT);
        self.bump(); // \left
        self.math_delim(LEFT_CMD);
        let body = self.open(SyntaxKind::MATH);
        loop {
            match self.kind() {
                None => {
                    self.error_at(opener, "unclosed `\\left`");
                    break;
                }
                Some(SyntaxKind::CONTROL_WORD) if self.at_command(RIGHT_CMD) => break,
                // Enclosing-scope closers: `\left … \right` cannot span a group,
                // math, or environment boundary, so hand the token back.
                Some(SyntaxKind::R_BRACE | SyntaxKind::DOLLAR) => {
                    self.error_at(opener, "unclosed `\\left`");
                    break;
                }
                Some(SyntaxKind::CONTROL_SYMBOL) if matches!(self.text(), "\\]" | "\\)") => {
                    self.error_at(opener, "unclosed `\\left`");
                    break;
                }
                Some(SyntaxKind::CONTROL_WORD) if self.at_env_end() => {
                    self.error_at(opener, "unclosed `\\left`");
                    break;
                }
                _ => {
                    if self.at_paragraph_break() {
                        self.error_at(opener, "unclosed `\\left`");
                        break;
                    }
                    self.math_element();
                }
            }
        }
        self.close(body);
        if self.at_command(RIGHT_CMD) {
            self.bump(); // \right
            self.math_delim(RIGHT_CMD);
        }
        self.close(pair);
    }

    /// Consume the single delimiter token following `\left`/`\right`: skip inline
    /// trivia (it rides as a direct child of the pair for losslessness; the
    /// formatter drops it), then take one token. The lexer has already isolated a
    /// word-character delimiter (`(`, `|`, `.`, …) into its own token, so a single
    /// `bump` suffices. A missing delimiter — the next meaningful token is a
    /// closer, another `\left`/`\right`, `\end`, a paragraph break, or EOF — is
    /// reported, not consumed.
    fn math_delim(&mut self, after: &str) {
        self.skip_trivia();
        let missing = match self.kind() {
            None | Some(SyntaxKind::R_BRACE | SyntaxKind::DOLLAR) => true,
            Some(SyntaxKind::CONTROL_SYMBOL) => matches!(self.text(), "\\]" | "\\)"),
            Some(SyntaxKind::CONTROL_WORD) => {
                self.at_env_end() || self.at_command(LEFT_CMD) || self.at_command(RIGHT_CMD)
            }
            _ => false,
        };
        if missing {
            self.error(format!("missing delimiter after `{after}`"));
            return;
        }
        self.bump();
    }

    /// A `\right` with no open `\left` (the math loop only reaches one here when
    /// it is unmatched). Report it and consume it with its delimiter so the parse
    /// stays lossless and makes progress.
    fn stray_right(&mut self) {
        debug_assert!(self.at_command(RIGHT_CMD));
        self.error("`\\right` without matching `\\left`");
        self.bump(); // \right
        self.math_delim(RIGHT_CMD);
    }

    /// The body of a named math environment (`equation`, `align`, `gather`, …): its
    /// atoms wrapped in a `MATH` node and parsed in math mode, exactly as `\[…\]`
    /// (see [`Self::delim_math`]) — so `^`/`_` build `SCRIPTED` nodes, the operator
    /// split fires, and `\left…\right` pair. Routed here for environments the
    /// signature data flags `math` ([`crate::parser::lexer::ParseCtx::is_math_environment`]).
    ///
    /// The terminator is the matching `\end` (or EOF), read via [`Self::at_block_end`]
    /// just like [`Self::parse_block`]; [`Self::finish_environment`] then consumes and
    /// name-checks it. Unlike `$`-math (where a `\end` is an *unclosed*-recovery
    /// anchor), `\end` is the normal, expected terminator here. A blank line inside the
    /// body stays trivia within the `MATH` node — no paragraph split — so losslessness
    /// holds. Progress is guaranteed: [`Self::math_element`] bumps trivia or descends
    /// into [`Self::math_scripted`], whose atom parser always consumes a token.
    pub(super) fn math_environment_body(&mut self) {
        let body = self.open(SyntaxKind::MATH);
        self.math_dollar.push(false);
        while !self.at_block_end(Block::Environment) {
            self.math_element();
        }
        self.math_dollar.pop();
        self.close(body);
    }
}