Skip to main content

fatou_parser/parser/
core.rs

1use crate::parser::ParseOptions;
2use crate::parser::context::ParserCtx;
3use crate::parser::events::Event;
4use crate::parser::expr::parse_stmt;
5use crate::parser::lexer::{TokKind, Token, lex};
6use crate::parser::tree_builder::{build_tree, syntax_kind_for};
7use crate::syntax::{SyntaxKind, SyntaxNode};
8
9pub use crate::parser::diagnostics::ParseDiagnostic;
10use crate::parser::diagnostics::{DiagnosticKind, push_diagnostic};
11
12/// The result of a parse: the lossless CST plus any diagnostics.
13#[derive(Debug, Clone)]
14pub struct ParseOutput {
15    pub cst: SyntaxNode,
16    pub diagnostics: Vec<ParseDiagnostic>,
17}
18
19/// Parse `text` into a lossless `rowan` CST. The drive loop works one logical
20/// line at a time: leading trivia (including newlines) is emitted directly at
21/// the root, then the line's statements are parsed with the Pratt parser. A line
22/// that carries a top-level `;` groups its statements into a `TOPLEVEL_SEMICOLON`
23/// node (mirroring JuliaSyntax's `toplevel-;`); a plain line stays bare. An
24/// unparseable token is consumed as one error token so the loop always makes
25/// progress.
26pub fn parse(text: &str) -> ParseOutput {
27    parse_with_options(text, ParseOptions::default())
28}
29
30/// Parse with an explicitly selected syntax edition.
31pub fn parse_with_options(text: &str, options: ParseOptions) -> ParseOutput {
32    let tokens = lex_for_options(text, options);
33    let ctx = ParserCtx::with_options(&tokens, options);
34    let mut diagnostics = Vec::new();
35    let mut events = Vec::new();
36
37    let mut i = 0usize;
38    while i < tokens.len() {
39        // Leading trivia and blank/newline runs belong directly at the root.
40        if tokens[i].kind.is_trivia() {
41            events.push(Event::Tok(i));
42            i += 1;
43            continue;
44        }
45
46        // Collect a single logical line (up to the next newline or EOF),
47        // tracking whether it carries a `;` separator. `leftover_mark` records
48        // the event offset right after the line's first complete statement so a
49        // trailing junk run on a separator-less line can be wrapped in an
50        // `ERROR` node and flagged with a `TrailingJunk` diagnostic (projected
51        // `(error-t …)`: `x y` ⇒ `x (error-t y)`).
52        let mut line = Vec::new();
53        let mut has_semicolon = false;
54        let mut leftover_mark: Option<usize> = None;
55        let mut first_is_doc_string = false;
56        // A doc-eligible string is only a docstring when a *real* statement
57        // follows it. If the leftover after the string can't start a statement
58        // (a stray closer/keyword: `"doc" ]`), the string is a plain statement
59        // and the remainder is junk — un-gate the raw trailing-junk collection.
60        let mut doc_no_target = false;
61        // A separator-less line: after the first complete statement, JuliaSyntax
62        // bumps the remainder as flat error tokens rather than re-parsing it. A
63        // line that carries a `;` keeps the per-segment behavior (deferred), so
64        // flat-junk collection is gated off when one is present.
65        let line_has_semicolon = rest_of_line_has_semicolon(&tokens, i);
66        while i < tokens.len() {
67            match tokens[i].kind {
68                TokKind::Newline => break,
69                TokKind::Semicolon => {
70                    has_semicolon = true;
71                    line.push(Event::Tok(i));
72                    i += 1;
73                }
74                k if k.is_trivia() => {
75                    line.push(Event::Tok(i));
76                    i += 1;
77                }
78                _ => {
79                    if !line_has_semicolon
80                        && leftover_mark.is_some()
81                        && (!first_is_doc_string || doc_no_target)
82                    {
83                        // Trailing junk after the first statement: collect raw
84                        // (no structural re-parse) so the wrapping below puts the
85                        // whole run in one ERROR node and the projector renders
86                        // delimiters, commas, and `@` as `✘` (`x y, z` ⇒
87                        // `x (error-t y ✘ z)`, `x@y` ⇒ `x (error-t ✘ y)`).
88                        line.push(Event::Tok(i));
89                        i += 1;
90                    } else if let Some(expr) = parse_stmt(&ctx, i, &mut diagnostics) {
91                        if leftover_mark.is_none() {
92                            first_is_doc_string = stmt_is_doc_string(&expr.events, &tokens);
93                        }
94                        line.extend(expr.events);
95                        i = expr.end;
96                        leftover_mark.get_or_insert(line.len());
97                    } else if leftover_mark.is_none()
98                        && is_close_delimiter_tok(tokens[i].kind)
99                        && !rest_of_line_has_semicolon(&tokens, i)
100                    {
101                        // A stray *closing* delimiter at statement start (no
102                        // preceding statement) is JuliaSyntax's leading empty
103                        // `(error)` plus an `(error-t ✘ …)` that swallows the
104                        // rest of the line: `)` ⇒ `(error) (error-t ✘)`,
105                        // `) x` ⇒ `(error) (error-t ✘ x)`, `)))` ⇒ `… ✘ ✘ ✘`.
106                        // The `;`-segment forms emit a subtler double marker, so
107                        // they are left to the loose-token fallback below.
108                        line.push(Event::Start(SyntaxKind::ERROR));
109                        line.push(Event::Finish);
110                        let run_start = tokens[i].start;
111                        line.push(Event::Start(SyntaxKind::ERROR));
112                        while i < tokens.len() && tokens[i].kind != TokKind::Newline {
113                            line.push(Event::Tok(i));
114                            i += 1;
115                        }
116                        line.push(Event::Finish);
117                        push_diagnostic(
118                            &mut diagnostics,
119                            DiagnosticKind::StrayCloser,
120                            "stray closing delimiter",
121                            run_start,
122                            tokens[i - 1].end,
123                        );
124                    } else if leftover_mark.is_none() && is_stray_block_keyword_tok(tokens[i].kind)
125                    {
126                        // A middle/closing block keyword (`end`, `else`,
127                        // `elseif`, `catch`, `finally`) where a statement is
128                        // expected is not a block opener; JuliaSyntax wraps it
129                        // alone in `(error <kw>)` and bumps the rest of the line
130                        // as a separate trailing-junk run (`end y z` ⇒
131                        // `(error end) (error-t y z)`). Emit the wrapped keyword
132                        // as the line's first "statement" and set `leftover_mark`
133                        // so the existing trailing-junk machinery handles the
134                        // remainder.
135                        line.push(Event::Start(SyntaxKind::ERROR));
136                        line.push(Event::Tok(i));
137                        line.push(Event::Finish);
138                        push_diagnostic(
139                            &mut diagnostics,
140                            DiagnosticKind::StrayKeyword,
141                            "unexpected block keyword",
142                            tokens[i].start,
143                            tokens[i].end,
144                        );
145                        i += 1;
146                        leftover_mark = Some(line.len());
147                    } else {
148                        // A leftover token that can't start a statement. After a
149                        // doc-eligible string this means the string has no
150                        // documentable target, so the rest of the line is junk.
151                        if first_is_doc_string && leftover_mark.is_some() {
152                            doc_no_target = true;
153                        }
154                        line.push(Event::Tok(i));
155                        i += 1;
156                    }
157                }
158            }
159        }
160
161        if has_semicolon {
162            events.push(Event::Start(SyntaxKind::TOPLEVEL_SEMICOLON));
163            events.extend(line);
164            events.push(Event::Finish);
165        } else if let Some(mark) = leftover_mark.filter(|&m| {
166            // Junk after the first statement on a separator-less line — but a
167            // bare docstring (`"a"\nfoo`) owns its trailing statement, so leave
168            // that to `fold_docstrings`. A docstring defers only when its
169            // leftover actually begins with a documentable statement subtree
170            // (`"doc" foo`); a leftover that opens with junk (`"doc" ]`) is
171            // recovered here so the stray closer becomes `(error-t ✘)`.
172            let tail = &line[m..];
173            let defer_to_doc = first_is_doc_string && leftover_starts_with_subtree(tail, &tokens);
174            !defer_to_doc && tail.iter().any(|e| is_significant_event(e, &tokens))
175        }) {
176            events.extend(line[..mark].iter().cloned());
177            let tail = &line[mark..];
178            // Leading trivia stays outside the error node; everything from the
179            // first significant leftover token onward is the recovered run.
180            let lead = tail
181                .iter()
182                .take_while(|e| !is_significant_event(e, &tokens))
183                .count();
184            events.extend(tail[..lead].iter().cloned());
185            let first_junk = tail[lead..].iter().find_map(|e| match e {
186                Event::Tok(idx) => Some(*idx),
187                _ => None,
188            });
189            events.push(Event::Start(SyntaxKind::ERROR));
190            events.extend(tail[lead..].iter().cloned());
191            events.push(Event::Finish);
192            if let Some(idx) = first_junk {
193                push_diagnostic(
194                    &mut diagnostics,
195                    DiagnosticKind::TrailingJunk,
196                    "trailing tokens after statement",
197                    tokens[idx].start,
198                    tokens[idx].end,
199                );
200            }
201        } else {
202            events.extend(line);
203        }
204    }
205
206    finish_parse(&tokens, &events, diagnostics, options)
207}
208
209/// Reparse statements using the same body driver as their enclosing construct.
210/// Keeping the owner in the temporary tree lets the flag passes recognize
211/// context-sensitive forms such as a struct's bare `const` fields.
212pub(super) fn parse_block_fragment(
213    text: &str,
214    owner: SyntaxKind,
215    options: ParseOptions,
216) -> Option<ParseOutput> {
217    let tokens = lex_for_options(text, options);
218    let ctx = ParserCtx::with_options(&tokens, options);
219    let mut diagnostics = Vec::new();
220    let mut events = vec![Event::Start(owner)];
221    let end = super::structural::run_block_fragment(&ctx, &mut events, owner, &mut diagnostics)?;
222    if end != tokens.len() {
223        return None;
224    }
225    events.push(Event::Finish);
226    let mut output = finish_parse(&tokens, &events, diagnostics, options);
227    output.cst = output.cst.first_child()?.first_child()?;
228    Some(output)
229}
230
231fn finish_parse(
232    tokens: &[Token],
233    events: &[Event],
234    mut diagnostics: Vec<ParseDiagnostic>,
235    options: ParseOptions,
236) -> ParseOutput {
237    let events = fold_docstrings(events, tokens);
238
239    let cst = build_tree(tokens, &events);
240    // The post-build flag passes. Each walks the finished CST for a shape
241    // JuliaSyntax error-wraps and records a *zero-width point* diagnostic at the
242    // offending node's start; the projector reconstructs the error wrapper from
243    // that anchor, so the CST itself stays faithful to the source.
244    flag_invalid_const_decls(&cst, &mut diagnostics);
245    flag_invalid_function_signatures(&cst, &mut diagnostics);
246    flag_invalid_catch_vars(&cst, &mut diagnostics);
247    flag_invalid_export_items(&cst, &mut diagnostics);
248    flag_numeric_overflow(&cst, &mut diagnostics);
249    if options.syntax_edition == crate::parser::SyntaxEdition::Julia113 {
250        for token in cst
251            .descendants_with_tokens()
252            .filter_map(|el| el.into_token())
253        {
254            if matches!(
255                token.kind(),
256                SyntaxKind::PLUS_PERCENT
257                    | SyntaxKind::MINUS_PERCENT
258                    | SyntaxKind::STAR_PERCENT
259                    | SyntaxKind::PLUS_PERCENT_EQ
260                    | SyntaxKind::MINUS_PERCENT_EQ
261                    | SyntaxKind::STAR_PERCENT_EQ
262                    | SyntaxKind::DOT_PLUS_PERCENT
263                    | SyntaxKind::DOT_MINUS_PERCENT
264                    | SyntaxKind::DOT_STAR_PERCENT
265                    | SyntaxKind::DOT_PLUS_PERCENT_EQ
266                    | SyntaxKind::DOT_MINUS_PERCENT_EQ
267                    | SyntaxKind::DOT_STAR_PERCENT_EQ
268            ) && !token
269                .text()
270                .chars()
271                .next_back()
272                .is_some_and(super::lexer::is_op_suffix_char)
273            {
274                let at = usize::from(token.text_range().start());
275                push_diagnostic(
276                    &mut diagnostics,
277                    DiagnosticKind::UnsupportedEdition,
278                    "wrapping arithmetic requires syntax edition 1.14",
279                    at,
280                    at,
281                );
282            }
283        }
284    }
285    ParseOutput { cst, diagnostics }
286}
287
288/// Flag decimal floats that Julia rejects instead of accepting as infinity.
289/// Rust's parsers return an infinite value for these spellings, which gives us
290/// the same range check without evaluating Julia code.
291fn flag_numeric_overflow(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
292    for literal in cst
293        .descendants()
294        .filter(|node| node.kind() == SyntaxKind::LITERAL)
295    {
296        let Some(number) = literal
297            .children_with_tokens()
298            .filter_map(|element| element.into_token())
299            .find(|token| matches!(token.kind(), SyntaxKind::FLOAT | SyntaxKind::FLOAT32))
300        else {
301            continue;
302        };
303        if !decimal_float_overflows(number.text(), number.kind()) {
304            continue;
305        }
306        let range = literal.text_range();
307        push_diagnostic(
308            diagnostics,
309            DiagnosticKind::NumericOverflow,
310            "overflow in floating point literal",
311            usize::from(range.start()),
312            usize::from(range.end()),
313        );
314    }
315}
316
317fn decimal_float_overflows(text: &str, kind: SyntaxKind) -> bool {
318    if text.contains("0x") || text.contains("0X") {
319        return false;
320    }
321    let mut parseable = text.replace('_', "");
322    if kind == SyntaxKind::FLOAT32
323        && let Some(marker) = parseable.find(['f', 'F'])
324    {
325        parseable.replace_range(marker..=marker, "e");
326    }
327    match kind {
328        SyntaxKind::FLOAT32 => parseable
329            .parse::<f32>()
330            .is_ok_and(|value| !value.is_finite()),
331        SyntaxKind::FLOAT => parseable
332            .parse::<f64>()
333            .is_ok_and(|value| !value.is_finite()),
334        _ => false,
335    }
336}
337
338/// Flag each parenthesized `export` item that is not a single symbol. An export
339/// item may be parenthesized, but only around a single identifier, operator,
340/// `var"…"`, or `$`-interpolation (`export (x)`/`export (+)`/`export ($a)`),
341/// which JuliaSyntax unwraps to the bare symbol. Any other parenthesized form
342/// (`export (x::T)`, `export (x, y)`, `export ()`, `export ((x))`) is invalid and
343/// JuliaSyntax error-wraps the parsed expression (`(export (error (::-i x T)))`).
344fn flag_invalid_export_items(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
345    for stmt in cst
346        .descendants()
347        .filter(|n| n.kind() == SyntaxKind::EXPORT_STMT)
348    {
349        for item in stmt
350            .children()
351            .filter(|c| matches!(c.kind(), SyntaxKind::PAREN_EXPR | SyntaxKind::TUPLE_EXPR))
352        {
353            if !export_paren_is_symbol(&item) {
354                let pos = usize::from(item.text_range().start());
355                push_diagnostic(
356                    diagnostics,
357                    DiagnosticKind::InvalidExportItem,
358                    "invalid `export` item: expected a single symbol",
359                    pos,
360                    pos,
361                );
362            }
363        }
364    }
365}
366
367/// Whether a parenthesized `export` item wraps a single symbol that JuliaSyntax
368/// unwraps: a lone identifier (`(x)`), `var"…"` (`(var"x")`), `$`-interpolation
369/// (`($a)`), or operator (`(+)`). A `(x, y)` tuple, an empty `()`, a nested paren
370/// (`((x))`), a literal (`(1)`), or any compound expression (`(x::T)`, `(a+b)`)
371/// is not. A `TUPLE_EXPR` (a comma/empty paren) is always invalid.
372fn export_paren_is_symbol(item: &SyntaxNode) -> bool {
373    if item.kind() != SyntaxKind::PAREN_EXPR {
374        return false;
375    }
376    let mut significant = item.children_with_tokens().filter(|el| {
377        !matches!(
378            el.kind(),
379            SyntaxKind::WHITESPACE
380                | SyntaxKind::NEWLINE
381                | SyntaxKind::COMMENT
382                | SyntaxKind::BLOCK_COMMENT
383                | SyntaxKind::LPAREN
384                | SyntaxKind::RPAREN
385        )
386    });
387    let Some(first) = significant.next() else {
388        return false;
389    };
390    if significant.next().is_some() {
391        return false;
392    }
393    match first {
394        // A symbol-shaped node: a bare name, `var"…"`, or `$`-interpolation. A
395        // `LITERAL`, `TYPE_ANNOTATION`, nested `PAREN_EXPR`, etc. is not.
396        rowan::NodeOrToken::Node(n) => match n.kind() {
397            SyntaxKind::NAME | SyntaxKind::NONSTANDARD_IDENTIFIER | SyntaxKind::INTERPOLATION => {
398                true
399            }
400            // A parenthesized operator value (`(..)`, `(√)`, `(⊕)`) — which
401            // `parse_paren` wraps in an `OPERATOR_ATOM` — is a valid export
402            // symbol, but a dotted broadcast operator (`(.+)`, `(.≤)`) is not:
403            // JuliaSyntax error-wraps it (`export (.+)` ⇒ `(export (error (. +)))`).
404            SyntaxKind::OPERATOR_ATOM => !export_operator_atom_is_dotted(&n),
405            _ => false,
406        },
407        // The only bare token a `PAREN_EXPR` carries is the lone operator-value
408        // form (`(+)`, `(:)`), which `parse_paren` builds for `is_paren_value_op`.
409        rowan::NodeOrToken::Token(_) => true,
410    }
411}
412
413/// Whether a parenthesized `export`/`import` operator atom is a *dotted*
414/// (broadcast) operator (`.+`, `.≤`, `.&&`), which JuliaSyntax rejects as an
415/// export symbol. The range/splat operators `..`/`...` lead with a *doubled*
416/// dot and are not broadcasts, so they are valid; a lone `√`/`⊕` has no dot.
417fn export_operator_atom_is_dotted(atom: &SyntaxNode) -> bool {
418    atom.children_with_tokens()
419        .filter_map(|el| el.into_token())
420        .find(|t| {
421            !matches!(
422                t.kind(),
423                SyntaxKind::WHITESPACE
424                    | SyntaxKind::NEWLINE
425                    | SyntaxKind::COMMENT
426                    | SyntaxKind::BLOCK_COMMENT
427            )
428        })
429        .is_some_and(|t| {
430            let bytes = t.text().as_bytes();
431            bytes.first() == Some(&b'.') && bytes.len() > 1 && bytes[1] != b'.'
432        })
433}
434
435/// Flag each `const` whose declaration is not the plain `=` assignment
436/// JuliaSyntax requires. A bare declaration (`const x`), a non-`=` assignment
437/// (`const x += 1`, `const x .= 1`), or a `global`/`local` declaration without an
438/// `=` (`const global x`) is invalid, so JuliaSyntax wraps the whole `const` in
439/// `(error …)` (`const x` ⇒ `(error (const x))`). A valid `const x = 1` — or a
440/// `global`/`local`-wrapped `=` (`const global x = 1`) — is left alone. The
441/// diagnostic anchors at the `const` keyword start.
442fn flag_invalid_const_decls(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
443    for node in cst
444        .descendants()
445        .filter(|n| n.kind() == SyntaxKind::CONST_STMT)
446    {
447        if !const_decl_is_assignment(&node) && !is_struct_const_field(&node) {
448            let pos = usize::from(node.text_range().start());
449            push_diagnostic(
450                diagnostics,
451                DiagnosticKind::ConstNotAssignment,
452                "expected assignment after `const`",
453                pos,
454                pos,
455            );
456        }
457    }
458}
459
460/// Flag each invalid `function`/`macro` signature. A bare name is a valid forward
461/// declaration only with a truly empty body; a direct parenthesized macro call or
462/// interpolation is an invalid anonymous-function parameter tuple. JuliaSyntax
463/// error-wraps either signature, and the diagnostic anchors at `SIGNATURE`.
464fn flag_invalid_function_signatures(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
465    for node in cst
466        .descendants()
467        .filter(|n| matches!(n.kind(), SyntaxKind::FUNCTION_DEF | SyntaxKind::MACRO_DEF))
468    {
469        let Some(sig) = node.children().find(|c| c.kind() == SyntaxKind::SIGNATURE) else {
470            continue;
471        };
472        let sig_is_bare_name = sig
473            .first_child()
474            .is_some_and(|inner| is_bare_signature_name(&inner));
475        let invalid_bare_name = sig_is_bare_name
476            && (!function_body_is_empty(&node) || invalid_forward_operator_name(&sig));
477        let invalid_parenthesized_macro = invalid_parenthesized_macro_signature(&sig);
478        if invalid_bare_name || invalid_parenthesized_macro {
479            let pos = usize::from(sig.text_range().start());
480            push_diagnostic(
481                diagnostics,
482                DiagnosticKind::InvalidFunctionSignature,
483                if invalid_bare_name {
484                    "invalid function signature: expected a call, not a bare name"
485                } else {
486                    "invalid function signature: macro call or interpolation cannot bind a parameter"
487                },
488                pos,
489                pos,
490            );
491        }
492    }
493}
494
495/// Whether a singleton anonymous-function parameter list is a macro call or an
496/// interpolation. These expression forms cannot bind a parameter, so JuliaSyntax
497/// wraps the whole tuple in an error. A comma, semicolon, or splat makes the
498/// corresponding tuple shape valid and therefore stays outside this recovery.
499fn invalid_parenthesized_macro_signature(signature: &SyntaxNode) -> bool {
500    let Some(tuple) = signature
501        .first_child()
502        .filter(|node| node.kind() == SyntaxKind::TUPLE_EXPR)
503    else {
504        return false;
505    };
506    if tuple.children().count() != 1
507        || tuple
508            .children_with_tokens()
509            .any(|element| element.kind() == SyntaxKind::COMMA)
510    {
511        return false;
512    }
513
514    let mut child = tuple.first_child();
515    while child
516        .as_ref()
517        .is_some_and(|node| node.kind() == SyntaxKind::PAREN_EXPR)
518    {
519        child = child.and_then(|node| node.first_child());
520    }
521    child.is_some_and(|node| {
522        matches!(
523            node.kind(),
524            SyntaxKind::MACRO_CALL | SyntaxKind::INTERPOLATION
525        )
526    })
527}
528
529/// JuliaSyntax 1.0 rejects syntactic, broadcast, and bare `&` operators as
530/// forward-declaration names even though ordinary operators remain valid.
531fn invalid_forward_operator_name(signature: &SyntaxNode) -> bool {
532    let Some(inner) = signature.first_child() else {
533        return false;
534    };
535    if inner.kind() == SyntaxKind::ERROR {
536        return true;
537    }
538    if inner.kind() != SyntaxKind::OPERATOR_ATOM {
539        return false;
540    }
541    inner.first_token().is_some_and(|token| {
542        let text = token.text();
543        text == "&" || (text.starts_with('.') && !text.starts_with(".."))
544    })
545}
546
547/// Whether a `function`/`macro` signature's first node is a *bare name* — the
548/// thing being declared rather than a call or other expression. Besides a plain
549/// or `var"…"` identifier and a `$`-interpolation, an operator name counts
550/// (`function + end`, `function .+ end`), including one Julia rejects as a name,
551/// which the parser leaves as an `ERROR`-wrapped operator atom (`function = end`
552/// ⇒ `(function (error =))`).
553pub(super) fn is_bare_signature_name(node: &SyntaxNode) -> bool {
554    match node.kind() {
555        SyntaxKind::NAME
556        | SyntaxKind::NONSTANDARD_IDENTIFIER
557        | SyntaxKind::INTERPOLATION
558        | SyntaxKind::OPERATOR_ATOM => true,
559        SyntaxKind::ERROR => node
560            .first_child()
561            .is_some_and(|inner| inner.kind() == SyntaxKind::OPERATOR_ATOM),
562        _ => false,
563    }
564}
565
566/// Flag each `catch` clause whose variable is not a plain identifier. The catch
567/// variable must be a bare identifier (`catch e`), a `$`-interpolation
568/// (`catch $e`), or a `var"…"` non-standard identifier (`catch var"e"`); any
569/// other expression (`catch e+3`, `catch e.f`, `catch f(e)`, `catch 3`) is
570/// invalid and JuliaSyntax wraps it in `(error …)` (`catch e+3` ⇒ `(catch
571/// (error (call-i e + 3)) …)`). The diagnostic anchors at the catch-variable
572/// node's start.
573fn flag_invalid_catch_vars(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
574    for node in cst
575        .descendants()
576        .filter(|n| n.kind() == SyntaxKind::CATCH_CLAUSE)
577    {
578        let Some(var) = node.children().find(|c| c.kind() != SyntaxKind::BLOCK) else {
579            continue;
580        };
581        if !matches!(
582            var.kind(),
583            SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::NONSTANDARD_IDENTIFIER
584        ) {
585            let pos = usize::from(var.text_range().start());
586            push_diagnostic(
587                diagnostics,
588                DiagnosticKind::CatchVarNotIdentifier,
589                "catch variable must be an identifier",
590                pos,
591                pos,
592            );
593        }
594    }
595}
596
597/// Whether a `function`/`macro`'s body is syntactically empty — no statement
598/// nodes and no `;` separator opening the block. A bare-name header keeps the
599/// forward-declaration form only while this holds; a `;` (`function f; end`) or
600/// any body statement marks the block as genuinely opened.
601fn function_body_is_empty(node: &SyntaxNode) -> bool {
602    match node.children().find(|c| c.kind() == SyntaxKind::BLOCK) {
603        Some(block) => {
604            block.first_child().is_none()
605                && !block
606                    .children_with_tokens()
607                    .any(|el| el.kind() == SyntaxKind::SEMICOLON)
608        }
609        None => true,
610    }
611}
612
613/// Whether `node` is a `const` field declaration directly inside a struct body
614/// (`struct A const a end` ⇒ `(const a)`, not wrapped). Such a bare `const` is
615/// valid; the exemption is narrow — a `const` nested inside an `if`/`begin`
616/// within the struct (its block's parent is not the `STRUCT_DEF`) is still an
617/// error.
618fn is_struct_const_field(node: &SyntaxNode) -> bool {
619    node.parent()
620        .filter(|b| b.kind() == SyntaxKind::BLOCK)
621        .and_then(|b| b.parent())
622        .is_some_and(|p| p.kind() == SyntaxKind::STRUCT_DEF)
623}
624
625/// Whether a `const`'s declaration is a plain `=` assignment. The body is the
626/// `const`'s first child node, unwrapping any `global`/`local` declaration; it is
627/// valid only when it is an `ASSIGNMENT_EXPR` headed by a plain `=` (not
628/// `.=`/`+=`/…).
629fn const_decl_is_assignment(node: &SyntaxNode) -> bool {
630    let mut body = node.first_child();
631    while let Some(n) = body {
632        match n.kind() {
633            SyntaxKind::GLOBAL_STMT | SyntaxKind::LOCAL_STMT => body = n.first_child(),
634            SyntaxKind::ASSIGNMENT_EXPR => {
635                return n
636                    .children_with_tokens()
637                    .filter_map(|el| el.into_token())
638                    .any(|t| t.kind() == SyntaxKind::EQ);
639            }
640            _ => return false,
641        }
642    }
643    false
644}
645
646/// Whether `kind` is a closing delimiter (`)`, `]`, `}`), which when stray at
647/// statement start drives JuliaSyntax's leading-`(error)` recovery.
648fn is_close_delimiter_tok(kind: TokKind) -> bool {
649    matches!(kind, TokKind::RParen | TokKind::RBracket | TokKind::RBrace)
650}
651
652/// Whether `kind` is a middle/closing block keyword (`end`, `else`, `elseif`,
653/// `catch`, `finally`) — a token that only closes or continues an enclosing
654/// block. Where a statement is expected one of these is stray, so JuliaSyntax
655/// wraps it alone in `(error <kw>)` (`@doc x\nend` ⇒ `(macrocall @doc x) (error
656/// end)`, `end y z` ⇒ `(error end) (error-t y z)`).
657fn is_stray_block_keyword_tok(kind: TokKind) -> bool {
658    matches!(
659        kind,
660        TokKind::EndKw
661            | TokKind::ElseKw
662            | TokKind::ElseifKw
663            | TokKind::CatchKw
664            | TokKind::FinallyKw
665    )
666}
667
668/// Whether the logical line starting at `start` (up to the next newline or EOF)
669/// carries a top-level `;`. The stray-closer recovery only applies to the clean
670/// separator-less form; `;`-segment lines keep the loose-token fallback.
671fn rest_of_line_has_semicolon(tokens: &[Token], start: usize) -> bool {
672    tokens[start..]
673        .iter()
674        .take_while(|t| t.kind != TokKind::Newline)
675        .any(|t| t.kind == TokKind::Semicolon)
676}
677
678/// Whether an event carries significant (non-trivia) content: any node opener,
679/// or a non-trivia leaf token.
680fn is_significant_event(event: &Event, tokens: &[Token]) -> bool {
681    match event {
682        Event::Start(_) => true,
683        Event::Tok(idx) => !tokens[*idx].kind.is_trivia(),
684        Event::Finish => false,
685    }
686}
687
688/// Whether the first significant event in `tail` opens a node (a real statement
689/// subtree). A docstring's trailing leftover defers to `fold_docstrings` only
690/// when it begins with such a subtree (its documentable target); a leftover that
691/// opens with a bare junk token is recovered as trailing junk instead.
692fn leftover_starts_with_subtree(tail: &[Event], tokens: &[Token]) -> bool {
693    matches!(
694        tail.iter().find(|e| is_significant_event(e, tokens)),
695        Some(Event::Start(_))
696    )
697}
698
699/// Whether a statement's events open a bare (doc-eligible) `STRING_LITERAL` —
700/// the first inner token is not a `STRING_PREFIX`. Such a statement starts a
701/// potential docstring, so a trailing statement on the same logical line is left
702/// to `fold_docstrings` rather than wrapped as junk.
703pub(super) fn stmt_is_doc_string(events: &[Event], tokens: &[Token]) -> bool {
704    matches!(
705        events.first(),
706        Some(Event::Start(SyntaxKind::STRING_LITERAL))
707    ) && string_is_doc_eligible(&events[1..], tokens)
708}
709
710/// Whether a node of `kind` is a statement container in which the docstring fold
711/// applies (a string literal statement directly followed by another statement).
712fn is_doc_container(kind: SyntaxKind) -> bool {
713    matches!(kind, SyntaxKind::BLOCK | SyntaxKind::TOPLEVEL_SEMICOLON)
714}
715
716/// Whether a `STRING_LITERAL` subtree is a plain string (doc-eligible) rather
717/// than a prefixed string macro (`r"…"`, `b"…"`), which is a macro call in
718/// JuliaSyntax and never a docstring. Eligible iff its first token is not a
719/// `STRING_PREFIX`.
720fn string_is_doc_eligible(inner: &[Event], tokens: &[Token]) -> bool {
721    match inner.first() {
722        Some(Event::Tok(idx)) => syntax_kind_for(tokens[*idx].kind) != SyntaxKind::STRING_PREFIX,
723        _ => true,
724    }
725}
726
727/// Fold docstrings across the whole event stream. A bare unprefixed
728/// `STRING_LITERAL` statement immediately followed by another statement — at
729/// most one newline of intervening trivia, no `;` — folds into a `DOC` node
730/// `(doc str target)`, mirroring JuliaSyntax's `parse_docstring`. The pass
731/// descends into every subtree so nested blocks (function/module/begin bodies)
732/// fold too.
733///
734/// The fold only ever *inserts* a `Start(DOC)` before the string's `Start` and a
735/// matching `Finish` after the target's `Finish` — every other event keeps its
736/// order and its meaning. So rather than rebuilding the stream level by level,
737/// this marks the two insertion points per fold and splices them in with one
738/// final pass, which keeps the whole thing linear in the event count.
739fn fold_docstrings(events: &[Event], tokens: &[Token]) -> Vec<Event> {
740    let extents = subtree_extents(events);
741    let mut marks = DocMarks {
742        open: vec![false; events.len()],
743        close: vec![false; events.len()],
744        folds: 0,
745    };
746    mark_doc_folds(events, tokens, &extents, 0, events.len(), true, &mut marks);
747
748    if marks.folds == 0 {
749        return events.to_vec();
750    }
751
752    let mut out = Vec::with_capacity(events.len() + 2 * marks.folds);
753    for (i, event) in events.iter().enumerate() {
754        // A fold opens before a `Start` and closes after a `Finish`, so the two
755        // marks can never land on the same event and their order is unambiguous.
756        if marks.open[i] {
757            out.push(Event::Start(SyntaxKind::DOC));
758        }
759        out.push(event.clone());
760        if marks.close[i] {
761            out.push(Event::Finish);
762        }
763    }
764    out
765}
766
767/// Where each fold inserts its `DOC` wrapper: `open[i]` puts a `Start(DOC)`
768/// before event `i`, `close[i]` puts a `Finish` after it.
769struct DocMarks {
770    open: Vec<bool>,
771    close: Vec<bool>,
772    folds: usize,
773}
774
775/// For every `Start` in `events`, the index of its matching `Finish`; `usize::MAX`
776/// for any unmatched opener. One stack pass, so a later walk can skip a whole
777/// subtree in constant time instead of rescanning it for the matching close.
778fn subtree_extents(events: &[Event]) -> Vec<usize> {
779    let mut extents = vec![usize::MAX; events.len()];
780    let mut open: Vec<usize> = Vec::new();
781    for (i, event) in events.iter().enumerate() {
782        match event {
783            Event::Start(_) => open.push(i),
784            Event::Finish => {
785                if let Some(start) = open.pop() {
786                    extents[start] = i;
787                }
788            }
789            Event::Tok(_) => {}
790        }
791    }
792    extents
793}
794
795/// Walk the direct children of the level spanning `start..end`, recording a fold
796/// for each docstring pair and descending into every subtree. `is_container` says
797/// whether a fold may happen at *this* level.
798fn mark_doc_folds(
799    events: &[Event],
800    tokens: &[Token],
801    extents: &[usize],
802    start: usize,
803    end: usize,
804    is_container: bool,
805    marks: &mut DocMarks,
806) {
807    let mut i = start;
808    while i < end {
809        let Event::Start(kind) = events[i] else {
810            i += 1;
811            continue;
812        };
813        // An unmatched opener runs to the end of this level.
814        let stop = extents[i].min(end);
815        mark_doc_folds(
816            events,
817            tokens,
818            extents,
819            i + 1,
820            stop,
821            is_doc_container(kind),
822            marks,
823        );
824
825        if is_container
826            && kind == SyntaxKind::STRING_LITERAL
827            && string_is_doc_eligible(&events[i + 1..stop], tokens)
828            && let Some(target) = doc_target(events, tokens, stop + 1, end)
829        {
830            let target_end = extents[target].min(end);
831            // The target is consumed by this fold: it is documented *by* the
832            // string, so it never goes on to document what follows it. Descend
833            // into it here, then resume past it.
834            mark_doc_folds(
835                events,
836                tokens,
837                extents,
838                target + 1,
839                target_end,
840                is_doc_container(as_start_kind(&events[target])),
841                marks,
842            );
843            marks.open[i] = true;
844            marks.close[target_end] = true;
845            marks.folds += 1;
846            i = target_end + 1;
847            continue;
848        }
849        i = stop + 1;
850    }
851}
852
853/// The kind of a `Start` event. Only ever called on an event already matched as
854/// one, so the fallback kind is unreachable.
855fn as_start_kind(event: &Event) -> SyntaxKind {
856    match event {
857        Event::Start(kind) => *kind,
858        _ => SyntaxKind::ERROR,
859    }
860}
861
862/// Scanning forward from `from` within a level ending at `end`, the index of the
863/// `Start` of the statement a preceding docstring documents: the next subtree
864/// child, reachable across at most one newline of trivia and no `;`. `None` if no
865/// eligible target follows.
866fn doc_target(events: &[Event], tokens: &[Token], from: usize, end: usize) -> Option<usize> {
867    let mut newlines = 0;
868    let mut j = from;
869    while j < end {
870        match events[j] {
871            // An error-recovery node is never a documentable target (`"doc"\n]`
872            // ⇒ `(string) (error) (error-t ✘)`, not a `(doc …)`); the string is
873            // a plain statement.
874            Event::Start(SyntaxKind::ERROR) => return None,
875            Event::Start(_) => return Some(j),
876            Event::Tok(t) => {
877                let kind = tokens[t].kind;
878                if kind == TokKind::Newline {
879                    newlines += 1;
880                    if newlines > 1 {
881                        return None;
882                    }
883                } else if !kind.is_trivia() {
884                    return None;
885                }
886                j += 1;
887            }
888            // Only reachable on an unbalanced stream, where it ends the level.
889            Event::Finish => return None,
890        }
891    }
892    None
893}
894
895/// Round-trip the input through the parser: concatenating every token in the CST
896/// must reproduce the original text exactly (the losslessness invariant).
897pub fn reconstruct(text: &str) -> String {
898    parse(text)
899        .cst
900        .descendants_with_tokens()
901        .filter_map(|el| el.into_token())
902        .map(|tok| tok.text().to_string())
903        .collect()
904}
905
906/// Lexical shapes are independent of the edition; grammatical keyword
907/// classification happens in the parser context.
908pub(super) fn lex_for_options(text: &str, _options: ParseOptions) -> Vec<Token<'_>> {
909    lex(text)
910}
911
912#[cfg(test)]
913mod tests {
914    use super::*;
915
916    fn assert_lossless(input: &str) {
917        assert_eq!(reconstruct(input), input, "not lossless: {input:?}");
918    }
919
920    #[test]
921    fn lossless_corpus() {
922        for input in [
923            "",
924            "x = 1\n",
925            "y = a + b * c\n",
926            "f(a, b, c)\n",
927            "v[1]\n",
928            "a[end]\n",
929            "a[end - 1]\n",
930            "a[2:end]\n",
931            "m[end, end]\n",
932            "z = (a + b) / 2\n",
933            "function g(x)\n    x ^ 2\nend\n",
934            "if a >= b\n    a\nelseif c\n    c\nelse\n    b\nend\n",
935            "begin\n    a\n    b\nend\n",
936            "map(xs) do x\n    x + 1\nend\n",
937            "open(\"f\") do\n    read()\nend\n",
938            "# a comment\nx = 1  # trailing\n",
939            "#= block =#\nq = 2\n",
940            "obj.field\n",
941            "x::Int\n",
942            "Vector{T}\n",
943            "foo(x::T) where {T<:Number} = x\n",
944            "f(a, b; c=1, d=2)\n",
945            "g(args...; kwargs...)\n",
946            "y = a .+ b .* c\n",
947            "z = .-x\n",
948            "a .= f.(b, c)\n",
949            "t = (a, b)\n",
950            "u = (a,)\n",
951            "e = ()\n",
952            "nt = (x = 1, y = 2)\n",
953            "y = a ? b : c\n",
954            "z = a == b ? x + 1 : x - 1\n",
955            "w = a ? b : c ? d : e\n",
956            "v = [1, 2, 3]\n",
957            "e = []\n",
958            "m = [1 2; 3 4]\n",
959            "s = [1 +2]\n",
960            "c = [1; 2; 3]\n",
961            "q = [x for x in xs]\n",
962            "r = [x^2 for x in 1:10 if x > 2]\n",
963            "g = (x for x in xs)\n",
964            "a; b\n",
965            "a;;;b;;\n",
966            ";a\n",
967            "@assert nl > 1  # trailing comment\nnx = 2\n",
968            "@foo x  # c\n",
969            "x = @bind 🐶 html\"<input>\"\n",
970            "begin @m 10 end\n",
971        ] {
972            assert_lossless(input);
973        }
974    }
975
976    #[test]
977    fn groups_top_level_semicolon_line() {
978        let out = parse("a; b\nc\n");
979        let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
980        use crate::syntax::SyntaxKind::{NAME, TOPLEVEL_SEMICOLON};
981        assert_eq!(kinds, vec![TOPLEVEL_SEMICOLON, NAME]);
982    }
983
984    #[test]
985    fn builds_expected_top_level_shape() {
986        let out = parse("x = 1 + 2\n");
987        let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
988        assert_eq!(kinds, vec![crate::syntax::SyntaxKind::ASSIGNMENT_EXPR]);
989        assert!(out.diagnostics.is_empty());
990    }
991
992    #[test]
993    fn overflowing_decimal_floats_are_parse_errors() {
994        let out = parse("x=1e400\ny=1f400\nz=1e-4000\n");
995        assert_eq!(
996            out.diagnostics
997                .iter()
998                .map(|diagnostic| (
999                    diagnostic.start,
1000                    diagnostic.end,
1001                    diagnostic.message.as_str()
1002                ))
1003                .collect::<Vec<_>>(),
1004            vec![
1005                (2, 7, "overflow in floating point literal"),
1006                (10, 15, "overflow in floating point literal"),
1007            ]
1008        );
1009    }
1010}