Skip to main content

fatou_parser/parser/
core.rs

1use crate::parser::events::Event;
2use crate::parser::expr::parse_stmt;
3use crate::parser::lexer::{TokKind, Token, lex};
4use crate::parser::tree_builder::{build_tree, syntax_kind_for};
5use crate::syntax::{SyntaxKind, SyntaxNode};
6
7pub use crate::parser::diagnostics::ParseDiagnostic;
8use crate::parser::diagnostics::{DiagnosticKind, push_diagnostic};
9
10/// The result of a parse: the lossless CST plus any diagnostics.
11#[derive(Debug, Clone)]
12pub struct ParseOutput {
13    pub cst: SyntaxNode,
14    pub diagnostics: Vec<ParseDiagnostic>,
15}
16
17/// Parse `text` into a lossless `rowan` CST. The drive loop works one logical
18/// line at a time: leading trivia (including newlines) is emitted directly at
19/// the root, then the line's statements are parsed with the Pratt parser. A line
20/// that carries a top-level `;` groups its statements into a `TOPLEVEL_SEMICOLON`
21/// node (mirroring JuliaSyntax's `toplevel-;`); a plain line stays bare. An
22/// unparseable token is consumed as one error token so the loop always makes
23/// progress.
24pub fn parse(text: &str) -> ParseOutput {
25    let tokens = lex(text);
26    let mut diagnostics = Vec::new();
27    let mut events = Vec::new();
28
29    let mut i = 0usize;
30    while i < tokens.len() {
31        // Leading trivia and blank/newline runs belong directly at the root.
32        if tokens[i].kind.is_trivia() {
33            events.push(Event::Tok(i));
34            i += 1;
35            continue;
36        }
37
38        // Collect a single logical line (up to the next newline or EOF),
39        // tracking whether it carries a `;` separator. `leftover_mark` records
40        // the event offset right after the line's first complete statement so a
41        // trailing junk run on a separator-less line can be wrapped in an
42        // `ERROR` node and flagged with a `TrailingJunk` diagnostic (projected
43        // `(error-t …)`: `x y` ⇒ `x (error-t y)`).
44        let mut line = Vec::new();
45        let mut has_semicolon = false;
46        let mut leftover_mark: Option<usize> = None;
47        let mut first_is_doc_string = false;
48        // A doc-eligible string is only a docstring when a *real* statement
49        // follows it. If the leftover after the string can't start a statement
50        // (a stray closer/keyword: `"doc" ]`), the string is a plain statement
51        // and the remainder is junk — un-gate the raw trailing-junk collection.
52        let mut doc_no_target = false;
53        // A separator-less line: after the first complete statement, JuliaSyntax
54        // bumps the remainder as flat error tokens rather than re-parsing it. A
55        // line that carries a `;` keeps the per-segment behavior (deferred), so
56        // flat-junk collection is gated off when one is present.
57        let line_has_semicolon = rest_of_line_has_semicolon(&tokens, i);
58        while i < tokens.len() {
59            match tokens[i].kind {
60                TokKind::Newline => break,
61                TokKind::Semicolon => {
62                    has_semicolon = true;
63                    line.push(Event::Tok(i));
64                    i += 1;
65                }
66                k if k.is_trivia() => {
67                    line.push(Event::Tok(i));
68                    i += 1;
69                }
70                _ => {
71                    if !line_has_semicolon
72                        && leftover_mark.is_some()
73                        && (!first_is_doc_string || doc_no_target)
74                    {
75                        // Trailing junk after the first statement: collect raw
76                        // (no structural re-parse) so the wrapping below puts the
77                        // whole run in one ERROR node and the projector renders
78                        // delimiters, commas, and `@` as `✘` (`x y, z` ⇒
79                        // `x (error-t y ✘ z)`, `x@y` ⇒ `x (error-t ✘ y)`).
80                        line.push(Event::Tok(i));
81                        i += 1;
82                    } else if let Some(expr) = parse_stmt(&tokens, i, &mut diagnostics) {
83                        if leftover_mark.is_none() {
84                            first_is_doc_string = stmt_is_doc_string(&expr.events, &tokens);
85                        }
86                        line.extend(expr.events);
87                        i = expr.end;
88                        leftover_mark.get_or_insert(line.len());
89                    } else if leftover_mark.is_none()
90                        && is_close_delimiter_tok(tokens[i].kind)
91                        && !rest_of_line_has_semicolon(&tokens, i)
92                    {
93                        // A stray *closing* delimiter at statement start (no
94                        // preceding statement) is JuliaSyntax's leading empty
95                        // `(error)` plus an `(error-t ✘ …)` that swallows the
96                        // rest of the line: `)` ⇒ `(error) (error-t ✘)`,
97                        // `) x` ⇒ `(error) (error-t ✘ x)`, `)))` ⇒ `… ✘ ✘ ✘`.
98                        // The `;`-segment forms emit a subtler double marker, so
99                        // they are left to the loose-token fallback below.
100                        line.push(Event::Start(SyntaxKind::ERROR));
101                        line.push(Event::Finish);
102                        let run_start = tokens[i].start;
103                        line.push(Event::Start(SyntaxKind::ERROR));
104                        while i < tokens.len() && tokens[i].kind != TokKind::Newline {
105                            line.push(Event::Tok(i));
106                            i += 1;
107                        }
108                        line.push(Event::Finish);
109                        push_diagnostic(
110                            &mut diagnostics,
111                            DiagnosticKind::StrayCloser,
112                            "stray closing delimiter",
113                            run_start,
114                            tokens[i - 1].end,
115                        );
116                    } else if leftover_mark.is_none() && is_stray_block_keyword_tok(tokens[i].kind)
117                    {
118                        // A middle/closing block keyword (`end`, `else`,
119                        // `elseif`, `catch`, `finally`) where a statement is
120                        // expected is not a block opener; JuliaSyntax wraps it
121                        // alone in `(error <kw>)` and bumps the rest of the line
122                        // as a separate trailing-junk run (`end y z` ⇒
123                        // `(error end) (error-t y z)`). Emit the wrapped keyword
124                        // as the line's first "statement" and set `leftover_mark`
125                        // so the existing trailing-junk machinery handles the
126                        // remainder.
127                        line.push(Event::Start(SyntaxKind::ERROR));
128                        line.push(Event::Tok(i));
129                        line.push(Event::Finish);
130                        push_diagnostic(
131                            &mut diagnostics,
132                            DiagnosticKind::StrayKeyword,
133                            "unexpected block keyword",
134                            tokens[i].start,
135                            tokens[i].end,
136                        );
137                        i += 1;
138                        leftover_mark = Some(line.len());
139                    } else {
140                        // A leftover token that can't start a statement. After a
141                        // doc-eligible string this means the string has no
142                        // documentable target, so the rest of the line is junk.
143                        if first_is_doc_string && leftover_mark.is_some() {
144                            doc_no_target = true;
145                        }
146                        line.push(Event::Tok(i));
147                        i += 1;
148                    }
149                }
150            }
151        }
152
153        if has_semicolon {
154            events.push(Event::Start(SyntaxKind::TOPLEVEL_SEMICOLON));
155            events.extend(line);
156            events.push(Event::Finish);
157        } else if let Some(mark) = leftover_mark.filter(|&m| {
158            // Junk after the first statement on a separator-less line — but a
159            // bare docstring (`"a"\nfoo`) owns its trailing statement, so leave
160            // that to `fold_docstrings`. A docstring defers only when its
161            // leftover actually begins with a documentable statement subtree
162            // (`"doc" foo`); a leftover that opens with junk (`"doc" ]`) is
163            // recovered here so the stray closer becomes `(error-t ✘)`.
164            let tail = &line[m..];
165            let defer_to_doc = first_is_doc_string && leftover_starts_with_subtree(tail, &tokens);
166            !defer_to_doc && tail.iter().any(|e| is_significant_event(e, &tokens))
167        }) {
168            events.extend(line[..mark].iter().cloned());
169            let tail = &line[mark..];
170            // Leading trivia stays outside the error node; everything from the
171            // first significant leftover token onward is the recovered run.
172            let lead = tail
173                .iter()
174                .take_while(|e| !is_significant_event(e, &tokens))
175                .count();
176            events.extend(tail[..lead].iter().cloned());
177            let first_junk = tail[lead..].iter().find_map(|e| match e {
178                Event::Tok(idx) => Some(*idx),
179                _ => None,
180            });
181            events.push(Event::Start(SyntaxKind::ERROR));
182            events.extend(tail[lead..].iter().cloned());
183            events.push(Event::Finish);
184            if let Some(idx) = first_junk {
185                push_diagnostic(
186                    &mut diagnostics,
187                    DiagnosticKind::TrailingJunk,
188                    "trailing tokens after statement",
189                    tokens[idx].start,
190                    tokens[idx].end,
191                );
192            }
193        } else {
194            events.extend(line);
195        }
196    }
197
198    let events = fold_docstrings(&events, &tokens);
199
200    let cst = build_tree(&tokens, &events);
201    // The post-build flag passes. Each walks the finished CST for a shape
202    // JuliaSyntax error-wraps and records a *zero-width point* diagnostic at the
203    // offending node's start; the projector reconstructs the error wrapper from
204    // that anchor, so the CST itself stays faithful to the source.
205    flag_invalid_const_decls(&cst, &mut diagnostics);
206    flag_invalid_function_signatures(&cst, &mut diagnostics);
207    flag_invalid_catch_vars(&cst, &mut diagnostics);
208    flag_invalid_export_items(&cst, &mut diagnostics);
209    flag_numeric_overflow(&cst, &mut diagnostics);
210    ParseOutput { cst, diagnostics }
211}
212
213/// Flag decimal floats that Julia rejects instead of accepting as infinity.
214/// Rust's parsers return an infinite value for these spellings, which gives us
215/// the same range check without evaluating Julia code.
216fn flag_numeric_overflow(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
217    for literal in cst
218        .descendants()
219        .filter(|node| node.kind() == SyntaxKind::LITERAL)
220    {
221        let Some(number) = literal
222            .children_with_tokens()
223            .filter_map(|element| element.into_token())
224            .find(|token| matches!(token.kind(), SyntaxKind::FLOAT | SyntaxKind::FLOAT32))
225        else {
226            continue;
227        };
228        if !decimal_float_overflows(number.text(), number.kind()) {
229            continue;
230        }
231        let range = literal.text_range();
232        push_diagnostic(
233            diagnostics,
234            DiagnosticKind::NumericOverflow,
235            "overflow in floating point literal",
236            usize::from(range.start()),
237            usize::from(range.end()),
238        );
239    }
240}
241
242fn decimal_float_overflows(text: &str, kind: SyntaxKind) -> bool {
243    if text.contains("0x") || text.contains("0X") {
244        return false;
245    }
246    let mut parseable = text.replace('_', "");
247    if kind == SyntaxKind::FLOAT32
248        && let Some(marker) = parseable.find(['f', 'F'])
249    {
250        parseable.replace_range(marker..=marker, "e");
251    }
252    match kind {
253        SyntaxKind::FLOAT32 => parseable
254            .parse::<f32>()
255            .is_ok_and(|value| !value.is_finite()),
256        SyntaxKind::FLOAT => parseable
257            .parse::<f64>()
258            .is_ok_and(|value| !value.is_finite()),
259        _ => false,
260    }
261}
262
263/// Flag each parenthesized `export` item that is not a single symbol. An export
264/// item may be parenthesized, but only around a single identifier, operator,
265/// `var"…"`, or `$`-interpolation (`export (x)`/`export (+)`/`export ($a)`),
266/// which JuliaSyntax unwraps to the bare symbol. Any other parenthesized form
267/// (`export (x::T)`, `export (x, y)`, `export ()`, `export ((x))`) is invalid and
268/// JuliaSyntax error-wraps the parsed expression (`(export (error (::-i x T)))`).
269fn flag_invalid_export_items(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
270    for stmt in cst
271        .descendants()
272        .filter(|n| n.kind() == SyntaxKind::EXPORT_STMT)
273    {
274        for item in stmt
275            .children()
276            .filter(|c| matches!(c.kind(), SyntaxKind::PAREN_EXPR | SyntaxKind::TUPLE_EXPR))
277        {
278            if !export_paren_is_symbol(&item) {
279                let pos = usize::from(item.text_range().start());
280                push_diagnostic(
281                    diagnostics,
282                    DiagnosticKind::InvalidExportItem,
283                    "invalid `export` item: expected a single symbol",
284                    pos,
285                    pos,
286                );
287            }
288        }
289    }
290}
291
292/// Whether a parenthesized `export` item wraps a single symbol that JuliaSyntax
293/// unwraps: a lone identifier (`(x)`), `var"…"` (`(var"x")`), `$`-interpolation
294/// (`($a)`), or operator (`(+)`). A `(x, y)` tuple, an empty `()`, a nested paren
295/// (`((x))`), a literal (`(1)`), or any compound expression (`(x::T)`, `(a+b)`)
296/// is not. A `TUPLE_EXPR` (a comma/empty paren) is always invalid.
297fn export_paren_is_symbol(item: &SyntaxNode) -> bool {
298    if item.kind() != SyntaxKind::PAREN_EXPR {
299        return false;
300    }
301    let mut significant = item.children_with_tokens().filter(|el| {
302        !matches!(
303            el.kind(),
304            SyntaxKind::WHITESPACE
305                | SyntaxKind::NEWLINE
306                | SyntaxKind::COMMENT
307                | SyntaxKind::BLOCK_COMMENT
308                | SyntaxKind::LPAREN
309                | SyntaxKind::RPAREN
310        )
311    });
312    let Some(first) = significant.next() else {
313        return false;
314    };
315    if significant.next().is_some() {
316        return false;
317    }
318    match first {
319        // A symbol-shaped node: a bare name, `var"…"`, or `$`-interpolation. A
320        // `LITERAL`, `TYPE_ANNOTATION`, nested `PAREN_EXPR`, etc. is not.
321        rowan::NodeOrToken::Node(n) => match n.kind() {
322            SyntaxKind::NAME | SyntaxKind::NONSTANDARD_IDENTIFIER | SyntaxKind::INTERPOLATION => {
323                true
324            }
325            // A parenthesized operator value (`(..)`, `(√)`, `(⊕)`) — which
326            // `parse_paren` wraps in an `OPERATOR_ATOM` — is a valid export
327            // symbol, but a dotted broadcast operator (`(.+)`, `(.≤)`) is not:
328            // JuliaSyntax error-wraps it (`export (.+)` ⇒ `(export (error (. +)))`).
329            SyntaxKind::OPERATOR_ATOM => !export_operator_atom_is_dotted(&n),
330            _ => false,
331        },
332        // The only bare token a `PAREN_EXPR` carries is the lone operator-value
333        // form (`(+)`, `(:)`), which `parse_paren` builds for `is_paren_value_op`.
334        rowan::NodeOrToken::Token(_) => true,
335    }
336}
337
338/// Whether a parenthesized `export`/`import` operator atom is a *dotted*
339/// (broadcast) operator (`.+`, `.≤`, `.&&`), which JuliaSyntax rejects as an
340/// export symbol. The range/splat operators `..`/`...` lead with a *doubled*
341/// dot and are not broadcasts, so they are valid; a lone `√`/`⊕` has no dot.
342fn export_operator_atom_is_dotted(atom: &SyntaxNode) -> bool {
343    atom.children_with_tokens()
344        .filter_map(|el| el.into_token())
345        .find(|t| {
346            !matches!(
347                t.kind(),
348                SyntaxKind::WHITESPACE
349                    | SyntaxKind::NEWLINE
350                    | SyntaxKind::COMMENT
351                    | SyntaxKind::BLOCK_COMMENT
352            )
353        })
354        .is_some_and(|t| {
355            let bytes = t.text().as_bytes();
356            bytes.first() == Some(&b'.') && bytes.len() > 1 && bytes[1] != b'.'
357        })
358}
359
360/// Flag each `const` whose declaration is not the plain `=` assignment
361/// JuliaSyntax requires. A bare declaration (`const x`), a non-`=` assignment
362/// (`const x += 1`, `const x .= 1`), or a `global`/`local` declaration without an
363/// `=` (`const global x`) is invalid, so JuliaSyntax wraps the whole `const` in
364/// `(error …)` (`const x` ⇒ `(error (const x))`). A valid `const x = 1` — or a
365/// `global`/`local`-wrapped `=` (`const global x = 1`) — is left alone. The
366/// diagnostic anchors at the `const` keyword start.
367fn flag_invalid_const_decls(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
368    for node in cst
369        .descendants()
370        .filter(|n| n.kind() == SyntaxKind::CONST_STMT)
371    {
372        if !const_decl_is_assignment(&node) && !is_struct_const_field(&node) {
373            let pos = usize::from(node.text_range().start());
374            push_diagnostic(
375                diagnostics,
376                DiagnosticKind::ConstNotAssignment,
377                "expected assignment after `const`",
378                pos,
379                pos,
380            );
381        }
382    }
383}
384
385/// Flag each invalid `function`/`macro` signature. A bare name is a valid forward
386/// declaration only with a truly empty body; a direct parenthesized macro call or
387/// interpolation is an invalid anonymous-function parameter tuple. JuliaSyntax
388/// error-wraps either signature, and the diagnostic anchors at `SIGNATURE`.
389fn flag_invalid_function_signatures(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
390    for node in cst
391        .descendants()
392        .filter(|n| matches!(n.kind(), SyntaxKind::FUNCTION_DEF | SyntaxKind::MACRO_DEF))
393    {
394        let Some(sig) = node.children().find(|c| c.kind() == SyntaxKind::SIGNATURE) else {
395            continue;
396        };
397        let sig_is_bare_name = sig
398            .first_child()
399            .is_some_and(|inner| is_bare_signature_name(&inner));
400        let invalid_bare_name = sig_is_bare_name
401            && (!function_body_is_empty(&node) || invalid_forward_operator_name(&sig));
402        let invalid_parenthesized_macro = invalid_parenthesized_macro_signature(&sig);
403        if invalid_bare_name || invalid_parenthesized_macro {
404            let pos = usize::from(sig.text_range().start());
405            push_diagnostic(
406                diagnostics,
407                DiagnosticKind::InvalidFunctionSignature,
408                if invalid_bare_name {
409                    "invalid function signature: expected a call, not a bare name"
410                } else {
411                    "invalid function signature: macro call or interpolation cannot bind a parameter"
412                },
413                pos,
414                pos,
415            );
416        }
417    }
418}
419
420/// Whether a singleton anonymous-function parameter list is a macro call or an
421/// interpolation. These expression forms cannot bind a parameter, so JuliaSyntax
422/// wraps the whole tuple in an error. A comma, semicolon, or splat makes the
423/// corresponding tuple shape valid and therefore stays outside this recovery.
424fn invalid_parenthesized_macro_signature(signature: &SyntaxNode) -> bool {
425    let Some(tuple) = signature
426        .first_child()
427        .filter(|node| node.kind() == SyntaxKind::TUPLE_EXPR)
428    else {
429        return false;
430    };
431    if tuple.children().count() != 1
432        || tuple
433            .children_with_tokens()
434            .any(|element| element.kind() == SyntaxKind::COMMA)
435    {
436        return false;
437    }
438
439    let mut child = tuple.first_child();
440    while child
441        .as_ref()
442        .is_some_and(|node| node.kind() == SyntaxKind::PAREN_EXPR)
443    {
444        child = child.and_then(|node| node.first_child());
445    }
446    child.is_some_and(|node| {
447        matches!(
448            node.kind(),
449            SyntaxKind::MACRO_CALL | SyntaxKind::INTERPOLATION
450        )
451    })
452}
453
454/// JuliaSyntax 1.0 rejects syntactic, broadcast, and bare `&` operators as
455/// forward-declaration names even though ordinary operators remain valid.
456fn invalid_forward_operator_name(signature: &SyntaxNode) -> bool {
457    let Some(inner) = signature.first_child() else {
458        return false;
459    };
460    if inner.kind() == SyntaxKind::ERROR {
461        return true;
462    }
463    if inner.kind() != SyntaxKind::OPERATOR_ATOM {
464        return false;
465    }
466    inner.first_token().is_some_and(|token| {
467        let text = token.text();
468        text == "&" || (text.starts_with('.') && !text.starts_with(".."))
469    })
470}
471
472/// Whether a `function`/`macro` signature's first node is a *bare name* — the
473/// thing being declared rather than a call or other expression. Besides a plain
474/// or `var"…"` identifier and a `$`-interpolation, an operator name counts
475/// (`function + end`, `function .+ end`), including one Julia rejects as a name,
476/// which the parser leaves as an `ERROR`-wrapped operator atom (`function = end`
477/// ⇒ `(function (error =))`).
478pub(super) fn is_bare_signature_name(node: &SyntaxNode) -> bool {
479    match node.kind() {
480        SyntaxKind::NAME
481        | SyntaxKind::NONSTANDARD_IDENTIFIER
482        | SyntaxKind::INTERPOLATION
483        | SyntaxKind::OPERATOR_ATOM => true,
484        SyntaxKind::ERROR => node
485            .first_child()
486            .is_some_and(|inner| inner.kind() == SyntaxKind::OPERATOR_ATOM),
487        _ => false,
488    }
489}
490
491/// Flag each `catch` clause whose variable is not a plain identifier. The catch
492/// variable must be a bare identifier (`catch e`), a `$`-interpolation
493/// (`catch $e`), or a `var"…"` non-standard identifier (`catch var"e"`); any
494/// other expression (`catch e+3`, `catch e.f`, `catch f(e)`, `catch 3`) is
495/// invalid and JuliaSyntax wraps it in `(error …)` (`catch e+3` ⇒ `(catch
496/// (error (call-i e + 3)) …)`). The diagnostic anchors at the catch-variable
497/// node's start.
498fn flag_invalid_catch_vars(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
499    for node in cst
500        .descendants()
501        .filter(|n| n.kind() == SyntaxKind::CATCH_CLAUSE)
502    {
503        let Some(var) = node.children().find(|c| c.kind() != SyntaxKind::BLOCK) else {
504            continue;
505        };
506        if !matches!(
507            var.kind(),
508            SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::NONSTANDARD_IDENTIFIER
509        ) {
510            let pos = usize::from(var.text_range().start());
511            push_diagnostic(
512                diagnostics,
513                DiagnosticKind::CatchVarNotIdentifier,
514                "catch variable must be an identifier",
515                pos,
516                pos,
517            );
518        }
519    }
520}
521
522/// Whether a `function`/`macro`'s body is syntactically empty — no statement
523/// nodes and no `;` separator opening the block. A bare-name header keeps the
524/// forward-declaration form only while this holds; a `;` (`function f; end`) or
525/// any body statement marks the block as genuinely opened.
526fn function_body_is_empty(node: &SyntaxNode) -> bool {
527    match node.children().find(|c| c.kind() == SyntaxKind::BLOCK) {
528        Some(block) => {
529            block.first_child().is_none()
530                && !block
531                    .children_with_tokens()
532                    .any(|el| el.kind() == SyntaxKind::SEMICOLON)
533        }
534        None => true,
535    }
536}
537
538/// Whether `node` is a `const` field declaration directly inside a struct body
539/// (`struct A const a end` ⇒ `(const a)`, not wrapped). Such a bare `const` is
540/// valid; the exemption is narrow — a `const` nested inside an `if`/`begin`
541/// within the struct (its block's parent is not the `STRUCT_DEF`) is still an
542/// error.
543fn is_struct_const_field(node: &SyntaxNode) -> bool {
544    node.parent()
545        .filter(|b| b.kind() == SyntaxKind::BLOCK)
546        .and_then(|b| b.parent())
547        .is_some_and(|p| p.kind() == SyntaxKind::STRUCT_DEF)
548}
549
550/// Whether a `const`'s declaration is a plain `=` assignment. The body is the
551/// `const`'s first child node, unwrapping any `global`/`local` declaration; it is
552/// valid only when it is an `ASSIGNMENT_EXPR` headed by a plain `=` (not
553/// `.=`/`+=`/…).
554fn const_decl_is_assignment(node: &SyntaxNode) -> bool {
555    let mut body = node.first_child();
556    while let Some(n) = body {
557        match n.kind() {
558            SyntaxKind::GLOBAL_STMT | SyntaxKind::LOCAL_STMT => body = n.first_child(),
559            SyntaxKind::ASSIGNMENT_EXPR => {
560                return n
561                    .children_with_tokens()
562                    .filter_map(|el| el.into_token())
563                    .any(|t| t.kind() == SyntaxKind::EQ);
564            }
565            _ => return false,
566        }
567    }
568    false
569}
570
571/// Whether `kind` is a closing delimiter (`)`, `]`, `}`), which when stray at
572/// statement start drives JuliaSyntax's leading-`(error)` recovery.
573fn is_close_delimiter_tok(kind: TokKind) -> bool {
574    matches!(kind, TokKind::RParen | TokKind::RBracket | TokKind::RBrace)
575}
576
577/// Whether `kind` is a middle/closing block keyword (`end`, `else`, `elseif`,
578/// `catch`, `finally`) — a token that only closes or continues an enclosing
579/// block. Where a statement is expected one of these is stray, so JuliaSyntax
580/// wraps it alone in `(error <kw>)` (`@doc x\nend` ⇒ `(macrocall @doc x) (error
581/// end)`, `end y z` ⇒ `(error end) (error-t y z)`).
582fn is_stray_block_keyword_tok(kind: TokKind) -> bool {
583    matches!(
584        kind,
585        TokKind::EndKw
586            | TokKind::ElseKw
587            | TokKind::ElseifKw
588            | TokKind::CatchKw
589            | TokKind::FinallyKw
590    )
591}
592
593/// Whether the logical line starting at `start` (up to the next newline or EOF)
594/// carries a top-level `;`. The stray-closer recovery only applies to the clean
595/// separator-less form; `;`-segment lines keep the loose-token fallback.
596fn rest_of_line_has_semicolon(tokens: &[Token], start: usize) -> bool {
597    tokens[start..]
598        .iter()
599        .take_while(|t| t.kind != TokKind::Newline)
600        .any(|t| t.kind == TokKind::Semicolon)
601}
602
603/// Whether an event carries significant (non-trivia) content: any node opener,
604/// or a non-trivia leaf token.
605fn is_significant_event(event: &Event, tokens: &[Token]) -> bool {
606    match event {
607        Event::Start(_) => true,
608        Event::Tok(idx) => !tokens[*idx].kind.is_trivia(),
609        Event::Finish => false,
610    }
611}
612
613/// Whether the first significant event in `tail` opens a node (a real statement
614/// subtree). A docstring's trailing leftover defers to `fold_docstrings` only
615/// when it begins with such a subtree (its documentable target); a leftover that
616/// opens with a bare junk token is recovered as trailing junk instead.
617fn leftover_starts_with_subtree(tail: &[Event], tokens: &[Token]) -> bool {
618    matches!(
619        tail.iter().find(|e| is_significant_event(e, tokens)),
620        Some(Event::Start(_))
621    )
622}
623
624/// Whether a statement's events open a bare (doc-eligible) `STRING_LITERAL` —
625/// the first inner token is not a `STRING_PREFIX`. Such a statement starts a
626/// potential docstring, so a trailing statement on the same logical line is left
627/// to `fold_docstrings` rather than wrapped as junk.
628pub(super) fn stmt_is_doc_string(events: &[Event], tokens: &[Token]) -> bool {
629    matches!(
630        events.first(),
631        Some(Event::Start(SyntaxKind::STRING_LITERAL))
632    ) && string_is_doc_eligible(&events[1..], tokens)
633}
634
635/// Whether a node of `kind` is a statement container in which the docstring fold
636/// applies (a string literal statement directly followed by another statement).
637fn is_doc_container(kind: SyntaxKind) -> bool {
638    matches!(kind, SyntaxKind::BLOCK | SyntaxKind::TOPLEVEL_SEMICOLON)
639}
640
641/// Whether a `STRING_LITERAL` subtree is a plain string (doc-eligible) rather
642/// than a prefixed string macro (`r"…"`, `b"…"`), which is a macro call in
643/// JuliaSyntax and never a docstring. Eligible iff its first token is not a
644/// `STRING_PREFIX`.
645fn string_is_doc_eligible(inner: &[Event], tokens: &[Token]) -> bool {
646    match inner.first() {
647        Some(Event::Tok(idx)) => syntax_kind_for(tokens[*idx].kind) != SyntaxKind::STRING_PREFIX,
648        _ => true,
649    }
650}
651
652/// Fold docstrings across the whole event stream. A bare unprefixed
653/// `STRING_LITERAL` statement immediately followed by another statement — at
654/// most one newline of intervening trivia, no `;` — folds into a `DOC` node
655/// `(doc str target)`, mirroring JuliaSyntax's `parse_docstring`. The pass
656/// descends into every subtree so nested blocks (function/module/begin bodies)
657/// fold too.
658///
659/// The fold only ever *inserts* a `Start(DOC)` before the string's `Start` and a
660/// matching `Finish` after the target's `Finish` — every other event keeps its
661/// order and its meaning. So rather than rebuilding the stream level by level,
662/// this marks the two insertion points per fold and splices them in with one
663/// final pass, which keeps the whole thing linear in the event count.
664fn fold_docstrings(events: &[Event], tokens: &[Token]) -> Vec<Event> {
665    let extents = subtree_extents(events);
666    let mut marks = DocMarks {
667        open: vec![false; events.len()],
668        close: vec![false; events.len()],
669        folds: 0,
670    };
671    mark_doc_folds(events, tokens, &extents, 0, events.len(), true, &mut marks);
672
673    if marks.folds == 0 {
674        return events.to_vec();
675    }
676
677    let mut out = Vec::with_capacity(events.len() + 2 * marks.folds);
678    for (i, event) in events.iter().enumerate() {
679        // A fold opens before a `Start` and closes after a `Finish`, so the two
680        // marks can never land on the same event and their order is unambiguous.
681        if marks.open[i] {
682            out.push(Event::Start(SyntaxKind::DOC));
683        }
684        out.push(event.clone());
685        if marks.close[i] {
686            out.push(Event::Finish);
687        }
688    }
689    out
690}
691
692/// Where each fold inserts its `DOC` wrapper: `open[i]` puts a `Start(DOC)`
693/// before event `i`, `close[i]` puts a `Finish` after it.
694struct DocMarks {
695    open: Vec<bool>,
696    close: Vec<bool>,
697    folds: usize,
698}
699
700/// For every `Start` in `events`, the index of its matching `Finish`; `usize::MAX`
701/// for any unmatched opener. One stack pass, so a later walk can skip a whole
702/// subtree in constant time instead of rescanning it for the matching close.
703fn subtree_extents(events: &[Event]) -> Vec<usize> {
704    let mut extents = vec![usize::MAX; events.len()];
705    let mut open: Vec<usize> = Vec::new();
706    for (i, event) in events.iter().enumerate() {
707        match event {
708            Event::Start(_) => open.push(i),
709            Event::Finish => {
710                if let Some(start) = open.pop() {
711                    extents[start] = i;
712                }
713            }
714            Event::Tok(_) => {}
715        }
716    }
717    extents
718}
719
720/// Walk the direct children of the level spanning `start..end`, recording a fold
721/// for each docstring pair and descending into every subtree. `is_container` says
722/// whether a fold may happen at *this* level.
723fn mark_doc_folds(
724    events: &[Event],
725    tokens: &[Token],
726    extents: &[usize],
727    start: usize,
728    end: usize,
729    is_container: bool,
730    marks: &mut DocMarks,
731) {
732    let mut i = start;
733    while i < end {
734        let Event::Start(kind) = events[i] else {
735            i += 1;
736            continue;
737        };
738        // An unmatched opener runs to the end of this level.
739        let stop = extents[i].min(end);
740        mark_doc_folds(
741            events,
742            tokens,
743            extents,
744            i + 1,
745            stop,
746            is_doc_container(kind),
747            marks,
748        );
749
750        if is_container
751            && kind == SyntaxKind::STRING_LITERAL
752            && string_is_doc_eligible(&events[i + 1..stop], tokens)
753            && let Some(target) = doc_target(events, tokens, stop + 1, end)
754        {
755            let target_end = extents[target].min(end);
756            // The target is consumed by this fold: it is documented *by* the
757            // string, so it never goes on to document what follows it. Descend
758            // into it here, then resume past it.
759            mark_doc_folds(
760                events,
761                tokens,
762                extents,
763                target + 1,
764                target_end,
765                is_doc_container(as_start_kind(&events[target])),
766                marks,
767            );
768            marks.open[i] = true;
769            marks.close[target_end] = true;
770            marks.folds += 1;
771            i = target_end + 1;
772            continue;
773        }
774        i = stop + 1;
775    }
776}
777
778/// The kind of a `Start` event. Only ever called on an event already matched as
779/// one, so the fallback kind is unreachable.
780fn as_start_kind(event: &Event) -> SyntaxKind {
781    match event {
782        Event::Start(kind) => *kind,
783        _ => SyntaxKind::ERROR,
784    }
785}
786
787/// Scanning forward from `from` within a level ending at `end`, the index of the
788/// `Start` of the statement a preceding docstring documents: the next subtree
789/// child, reachable across at most one newline of trivia and no `;`. `None` if no
790/// eligible target follows.
791fn doc_target(events: &[Event], tokens: &[Token], from: usize, end: usize) -> Option<usize> {
792    let mut newlines = 0;
793    let mut j = from;
794    while j < end {
795        match events[j] {
796            // An error-recovery node is never a documentable target (`"doc"\n]`
797            // ⇒ `(string) (error) (error-t ✘)`, not a `(doc …)`); the string is
798            // a plain statement.
799            Event::Start(SyntaxKind::ERROR) => return None,
800            Event::Start(_) => return Some(j),
801            Event::Tok(t) => {
802                let kind = tokens[t].kind;
803                if kind == TokKind::Newline {
804                    newlines += 1;
805                    if newlines > 1 {
806                        return None;
807                    }
808                } else if !kind.is_trivia() {
809                    return None;
810                }
811                j += 1;
812            }
813            // Only reachable on an unbalanced stream, where it ends the level.
814            Event::Finish => return None,
815        }
816    }
817    None
818}
819
820/// Round-trip the input through the parser: concatenating every token in the CST
821/// must reproduce the original text exactly (the losslessness invariant).
822pub fn reconstruct(text: &str) -> String {
823    parse(text)
824        .cst
825        .descendants_with_tokens()
826        .filter_map(|el| el.into_token())
827        .map(|tok| tok.text().to_string())
828        .collect()
829}
830
831#[cfg(test)]
832mod tests {
833    use super::*;
834
835    fn assert_lossless(input: &str) {
836        assert_eq!(reconstruct(input), input, "not lossless: {input:?}");
837    }
838
839    #[test]
840    fn lossless_corpus() {
841        for input in [
842            "",
843            "x = 1\n",
844            "y = a + b * c\n",
845            "f(a, b, c)\n",
846            "v[1]\n",
847            "a[end]\n",
848            "a[end - 1]\n",
849            "a[2:end]\n",
850            "m[end, end]\n",
851            "z = (a + b) / 2\n",
852            "function g(x)\n    x ^ 2\nend\n",
853            "if a >= b\n    a\nelseif c\n    c\nelse\n    b\nend\n",
854            "begin\n    a\n    b\nend\n",
855            "map(xs) do x\n    x + 1\nend\n",
856            "open(\"f\") do\n    read()\nend\n",
857            "# a comment\nx = 1  # trailing\n",
858            "#= block =#\nq = 2\n",
859            "obj.field\n",
860            "x::Int\n",
861            "Vector{T}\n",
862            "foo(x::T) where {T<:Number} = x\n",
863            "f(a, b; c=1, d=2)\n",
864            "g(args...; kwargs...)\n",
865            "y = a .+ b .* c\n",
866            "z = .-x\n",
867            "a .= f.(b, c)\n",
868            "t = (a, b)\n",
869            "u = (a,)\n",
870            "e = ()\n",
871            "nt = (x = 1, y = 2)\n",
872            "y = a ? b : c\n",
873            "z = a == b ? x + 1 : x - 1\n",
874            "w = a ? b : c ? d : e\n",
875            "v = [1, 2, 3]\n",
876            "e = []\n",
877            "m = [1 2; 3 4]\n",
878            "s = [1 +2]\n",
879            "c = [1; 2; 3]\n",
880            "q = [x for x in xs]\n",
881            "r = [x^2 for x in 1:10 if x > 2]\n",
882            "g = (x for x in xs)\n",
883            "a; b\n",
884            "a;;;b;;\n",
885            ";a\n",
886            "@assert nl > 1  # trailing comment\nnx = 2\n",
887            "@foo x  # c\n",
888            "x = @bind 🐶 html\"<input>\"\n",
889            "begin @m 10 end\n",
890        ] {
891            assert_lossless(input);
892        }
893    }
894
895    #[test]
896    fn groups_top_level_semicolon_line() {
897        let out = parse("a; b\nc\n");
898        let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
899        use crate::syntax::SyntaxKind::{NAME, TOPLEVEL_SEMICOLON};
900        assert_eq!(kinds, vec![TOPLEVEL_SEMICOLON, NAME]);
901    }
902
903    #[test]
904    fn builds_expected_top_level_shape() {
905        let out = parse("x = 1 + 2\n");
906        let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
907        assert_eq!(kinds, vec![crate::syntax::SyntaxKind::ASSIGNMENT_EXPR]);
908        assert!(out.diagnostics.is_empty());
909    }
910
911    #[test]
912    fn overflowing_decimal_floats_are_parse_errors() {
913        let out = parse("x=1e400\ny=1f400\nz=1e-4000\n");
914        assert_eq!(
915            out.diagnostics
916                .iter()
917                .map(|diagnostic| (
918                    diagnostic.start,
919                    diagnostic.end,
920                    diagnostic.message.as_str()
921                ))
922                .collect::<Vec<_>>(),
923            vec![
924                (2, 7, "overflow in floating point literal"),
925                (10, 15, "overflow in floating point literal"),
926            ]
927        );
928    }
929}