fatou_parser/parser/core.rs
1use crate::parser::ParseOptions;
2use crate::parser::context::ParserCtx;
3use crate::parser::events::Event;
4use crate::parser::expr::parse_stmt;
5use crate::parser::lexer::{TokKind, Token, lex};
6use crate::parser::tree_builder::{build_tree, syntax_kind_for};
7use crate::syntax::{SyntaxKind, SyntaxNode};
8
9pub use crate::parser::diagnostics::ParseDiagnostic;
10use crate::parser::diagnostics::{DiagnosticKind, push_diagnostic};
11
12/// The result of a parse: the lossless CST plus any diagnostics.
13#[derive(Debug, Clone)]
14pub struct ParseOutput {
15 pub cst: SyntaxNode,
16 pub diagnostics: Vec<ParseDiagnostic>,
17}
18
19/// Parse `text` into a lossless `rowan` CST. The drive loop works one logical
20/// line at a time: leading trivia (including newlines) is emitted directly at
21/// the root, then the line's statements are parsed with the Pratt parser. A line
22/// that carries a top-level `;` groups its statements into a `TOPLEVEL_SEMICOLON`
23/// node (mirroring JuliaSyntax's `toplevel-;`); a plain line stays bare. An
24/// unparseable token is consumed as one error token so the loop always makes
25/// progress.
26pub fn parse(text: &str) -> ParseOutput {
27 parse_with_options(text, ParseOptions::default())
28}
29
30/// Parse with an explicitly selected syntax edition.
31pub fn parse_with_options(text: &str, options: ParseOptions) -> ParseOutput {
32 let tokens = lex_for_options(text, options);
33 let ctx = ParserCtx::with_options(&tokens, options);
34 let mut diagnostics = Vec::new();
35 let mut events = Vec::new();
36
37 let mut i = 0usize;
38 while i < tokens.len() {
39 // Leading trivia and blank/newline runs belong directly at the root.
40 if tokens[i].kind.is_trivia() {
41 events.push(Event::Tok(i));
42 i += 1;
43 continue;
44 }
45
46 // Collect a single logical line (up to the next newline or EOF),
47 // tracking whether it carries a `;` separator. `leftover_mark` records
48 // the event offset right after the line's first complete statement so a
49 // trailing junk run on a separator-less line can be wrapped in an
50 // `ERROR` node and flagged with a `TrailingJunk` diagnostic (projected
51 // `(error-t …)`: `x y` ⇒ `x (error-t y)`).
52 let mut line = Vec::new();
53 let mut has_semicolon = false;
54 let mut leftover_mark: Option<usize> = None;
55 let mut first_is_doc_string = false;
56 // A doc-eligible string is only a docstring when a *real* statement
57 // follows it. If the leftover after the string can't start a statement
58 // (a stray closer/keyword: `"doc" ]`), the string is a plain statement
59 // and the remainder is junk — un-gate the raw trailing-junk collection.
60 let mut doc_no_target = false;
61 // A separator-less line: after the first complete statement, JuliaSyntax
62 // bumps the remainder as flat error tokens rather than re-parsing it. A
63 // line that carries a `;` keeps the per-segment behavior (deferred), so
64 // flat-junk collection is gated off when one is present.
65 let line_has_semicolon = rest_of_line_has_semicolon(&tokens, i);
66 while i < tokens.len() {
67 match tokens[i].kind {
68 TokKind::Newline => break,
69 TokKind::Semicolon => {
70 has_semicolon = true;
71 line.push(Event::Tok(i));
72 i += 1;
73 }
74 k if k.is_trivia() => {
75 line.push(Event::Tok(i));
76 i += 1;
77 }
78 _ => {
79 if !line_has_semicolon
80 && leftover_mark.is_some()
81 && (!first_is_doc_string || doc_no_target)
82 {
83 // Trailing junk after the first statement: collect raw
84 // (no structural re-parse) so the wrapping below puts the
85 // whole run in one ERROR node and the projector renders
86 // delimiters, commas, and `@` as `✘` (`x y, z` ⇒
87 // `x (error-t y ✘ z)`, `x@y` ⇒ `x (error-t ✘ y)`).
88 line.push(Event::Tok(i));
89 i += 1;
90 } else if let Some(expr) = parse_stmt(&ctx, i, &mut diagnostics) {
91 if leftover_mark.is_none() {
92 first_is_doc_string = stmt_is_doc_string(&expr.events, &tokens);
93 }
94 line.extend(expr.events);
95 i = expr.end;
96 leftover_mark.get_or_insert(line.len());
97 } else if leftover_mark.is_none()
98 && is_close_delimiter_tok(tokens[i].kind)
99 && !rest_of_line_has_semicolon(&tokens, i)
100 {
101 // A stray *closing* delimiter at statement start (no
102 // preceding statement) is JuliaSyntax's leading empty
103 // `(error)` plus an `(error-t ✘ …)` that swallows the
104 // rest of the line: `)` ⇒ `(error) (error-t ✘)`,
105 // `) x` ⇒ `(error) (error-t ✘ x)`, `)))` ⇒ `… ✘ ✘ ✘`.
106 // The `;`-segment forms emit a subtler double marker, so
107 // they are left to the loose-token fallback below.
108 line.push(Event::Start(SyntaxKind::ERROR));
109 line.push(Event::Finish);
110 let run_start = tokens[i].start;
111 line.push(Event::Start(SyntaxKind::ERROR));
112 while i < tokens.len() && tokens[i].kind != TokKind::Newline {
113 line.push(Event::Tok(i));
114 i += 1;
115 }
116 line.push(Event::Finish);
117 push_diagnostic(
118 &mut diagnostics,
119 DiagnosticKind::StrayCloser,
120 "stray closing delimiter",
121 run_start,
122 tokens[i - 1].end,
123 );
124 } else if leftover_mark.is_none() && is_stray_block_keyword_tok(tokens[i].kind)
125 {
126 // A middle/closing block keyword (`end`, `else`,
127 // `elseif`, `catch`, `finally`) where a statement is
128 // expected is not a block opener; JuliaSyntax wraps it
129 // alone in `(error <kw>)` and bumps the rest of the line
130 // as a separate trailing-junk run (`end y z` ⇒
131 // `(error end) (error-t y z)`). Emit the wrapped keyword
132 // as the line's first "statement" and set `leftover_mark`
133 // so the existing trailing-junk machinery handles the
134 // remainder.
135 line.push(Event::Start(SyntaxKind::ERROR));
136 line.push(Event::Tok(i));
137 line.push(Event::Finish);
138 push_diagnostic(
139 &mut diagnostics,
140 DiagnosticKind::StrayKeyword,
141 "unexpected block keyword",
142 tokens[i].start,
143 tokens[i].end,
144 );
145 i += 1;
146 leftover_mark = Some(line.len());
147 } else {
148 // A leftover token that can't start a statement. After a
149 // doc-eligible string this means the string has no
150 // documentable target, so the rest of the line is junk.
151 if first_is_doc_string && leftover_mark.is_some() {
152 doc_no_target = true;
153 }
154 line.push(Event::Tok(i));
155 i += 1;
156 }
157 }
158 }
159 }
160
161 if has_semicolon {
162 events.push(Event::Start(SyntaxKind::TOPLEVEL_SEMICOLON));
163 events.extend(line);
164 events.push(Event::Finish);
165 } else if let Some(mark) = leftover_mark.filter(|&m| {
166 // Junk after the first statement on a separator-less line — but a
167 // bare docstring (`"a"\nfoo`) owns its trailing statement, so leave
168 // that to `fold_docstrings`. A docstring defers only when its
169 // leftover actually begins with a documentable statement subtree
170 // (`"doc" foo`); a leftover that opens with junk (`"doc" ]`) is
171 // recovered here so the stray closer becomes `(error-t ✘)`.
172 let tail = &line[m..];
173 let defer_to_doc = first_is_doc_string && leftover_starts_with_subtree(tail, &tokens);
174 !defer_to_doc && tail.iter().any(|e| is_significant_event(e, &tokens))
175 }) {
176 events.extend(line[..mark].iter().cloned());
177 let tail = &line[mark..];
178 // Leading trivia stays outside the error node; everything from the
179 // first significant leftover token onward is the recovered run.
180 let lead = tail
181 .iter()
182 .take_while(|e| !is_significant_event(e, &tokens))
183 .count();
184 events.extend(tail[..lead].iter().cloned());
185 let first_junk = tail[lead..].iter().find_map(|e| match e {
186 Event::Tok(idx) => Some(*idx),
187 _ => None,
188 });
189 events.push(Event::Start(SyntaxKind::ERROR));
190 events.extend(tail[lead..].iter().cloned());
191 events.push(Event::Finish);
192 if let Some(idx) = first_junk {
193 push_diagnostic(
194 &mut diagnostics,
195 DiagnosticKind::TrailingJunk,
196 "trailing tokens after statement",
197 tokens[idx].start,
198 tokens[idx].end,
199 );
200 }
201 } else {
202 events.extend(line);
203 }
204 }
205
206 finish_parse(&tokens, &events, diagnostics, options)
207}
208
209/// Reparse statements using the same body driver as their enclosing construct.
210/// Keeping the owner in the temporary tree lets the flag passes recognize
211/// context-sensitive forms such as a struct's bare `const` fields.
212pub(super) fn parse_block_fragment(
213 text: &str,
214 owner: SyntaxKind,
215 options: ParseOptions,
216) -> Option<ParseOutput> {
217 let tokens = lex_for_options(text, options);
218 let ctx = ParserCtx::with_options(&tokens, options);
219 let mut diagnostics = Vec::new();
220 let mut events = vec![Event::Start(owner)];
221 let end = super::structural::run_block_fragment(&ctx, &mut events, owner, &mut diagnostics)?;
222 if end != tokens.len() {
223 return None;
224 }
225 events.push(Event::Finish);
226 let mut output = finish_parse(&tokens, &events, diagnostics, options);
227 output.cst = output.cst.first_child()?.first_child()?;
228 Some(output)
229}
230
231fn finish_parse(
232 tokens: &[Token],
233 events: &[Event],
234 mut diagnostics: Vec<ParseDiagnostic>,
235 options: ParseOptions,
236) -> ParseOutput {
237 let events = fold_docstrings(events, tokens);
238
239 let cst = build_tree(tokens, &events);
240 // The post-build flag passes. Each walks the finished CST for a shape
241 // JuliaSyntax error-wraps and records a *zero-width point* diagnostic at the
242 // offending node's start; the projector reconstructs the error wrapper from
243 // that anchor, so the CST itself stays faithful to the source.
244 flag_invalid_const_decls(&cst, &mut diagnostics);
245 flag_invalid_function_signatures(&cst, &mut diagnostics);
246 flag_invalid_catch_vars(&cst, &mut diagnostics);
247 flag_invalid_export_items(&cst, &mut diagnostics);
248 flag_numeric_overflow(&cst, &mut diagnostics);
249 if options.syntax_edition == crate::parser::SyntaxEdition::Julia113 {
250 for token in cst
251 .descendants_with_tokens()
252 .filter_map(|el| el.into_token())
253 {
254 if matches!(
255 token.kind(),
256 SyntaxKind::PLUS_PERCENT
257 | SyntaxKind::MINUS_PERCENT
258 | SyntaxKind::STAR_PERCENT
259 | SyntaxKind::PLUS_PERCENT_EQ
260 | SyntaxKind::MINUS_PERCENT_EQ
261 | SyntaxKind::STAR_PERCENT_EQ
262 | SyntaxKind::DOT_PLUS_PERCENT
263 | SyntaxKind::DOT_MINUS_PERCENT
264 | SyntaxKind::DOT_STAR_PERCENT
265 | SyntaxKind::DOT_PLUS_PERCENT_EQ
266 | SyntaxKind::DOT_MINUS_PERCENT_EQ
267 | SyntaxKind::DOT_STAR_PERCENT_EQ
268 ) && !token
269 .text()
270 .chars()
271 .next_back()
272 .is_some_and(super::lexer::is_op_suffix_char)
273 {
274 let at = usize::from(token.text_range().start());
275 push_diagnostic(
276 &mut diagnostics,
277 DiagnosticKind::UnsupportedEdition,
278 "wrapping arithmetic requires syntax edition 1.14",
279 at,
280 at,
281 );
282 }
283 }
284 }
285 ParseOutput { cst, diagnostics }
286}
287
288/// Flag decimal floats that Julia rejects instead of accepting as infinity.
289/// Rust's parsers return an infinite value for these spellings, which gives us
290/// the same range check without evaluating Julia code.
291fn flag_numeric_overflow(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
292 for literal in cst
293 .descendants()
294 .filter(|node| node.kind() == SyntaxKind::LITERAL)
295 {
296 let Some(number) = literal
297 .children_with_tokens()
298 .filter_map(|element| element.into_token())
299 .find(|token| matches!(token.kind(), SyntaxKind::FLOAT | SyntaxKind::FLOAT32))
300 else {
301 continue;
302 };
303 if !decimal_float_overflows(number.text(), number.kind()) {
304 continue;
305 }
306 let range = literal.text_range();
307 push_diagnostic(
308 diagnostics,
309 DiagnosticKind::NumericOverflow,
310 "overflow in floating point literal",
311 usize::from(range.start()),
312 usize::from(range.end()),
313 );
314 }
315}
316
317fn decimal_float_overflows(text: &str, kind: SyntaxKind) -> bool {
318 if text.contains("0x") || text.contains("0X") {
319 return false;
320 }
321 let mut parseable = text.replace('_', "");
322 if kind == SyntaxKind::FLOAT32
323 && let Some(marker) = parseable.find(['f', 'F'])
324 {
325 parseable.replace_range(marker..=marker, "e");
326 }
327 match kind {
328 SyntaxKind::FLOAT32 => parseable
329 .parse::<f32>()
330 .is_ok_and(|value| !value.is_finite()),
331 SyntaxKind::FLOAT => parseable
332 .parse::<f64>()
333 .is_ok_and(|value| !value.is_finite()),
334 _ => false,
335 }
336}
337
338/// Flag each parenthesized `export` item that is not a single symbol. An export
339/// item may be parenthesized, but only around a single identifier, operator,
340/// `var"…"`, or `$`-interpolation (`export (x)`/`export (+)`/`export ($a)`),
341/// which JuliaSyntax unwraps to the bare symbol. Any other parenthesized form
342/// (`export (x::T)`, `export (x, y)`, `export ()`, `export ((x))`) is invalid and
343/// JuliaSyntax error-wraps the parsed expression (`(export (error (::-i x T)))`).
344fn flag_invalid_export_items(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
345 for stmt in cst
346 .descendants()
347 .filter(|n| n.kind() == SyntaxKind::EXPORT_STMT)
348 {
349 for item in stmt
350 .children()
351 .filter(|c| matches!(c.kind(), SyntaxKind::PAREN_EXPR | SyntaxKind::TUPLE_EXPR))
352 {
353 if !export_paren_is_symbol(&item) {
354 let pos = usize::from(item.text_range().start());
355 push_diagnostic(
356 diagnostics,
357 DiagnosticKind::InvalidExportItem,
358 "invalid `export` item: expected a single symbol",
359 pos,
360 pos,
361 );
362 }
363 }
364 }
365}
366
367/// Whether a parenthesized `export` item wraps a single symbol that JuliaSyntax
368/// unwraps: a lone identifier (`(x)`), `var"…"` (`(var"x")`), `$`-interpolation
369/// (`($a)`), or operator (`(+)`). A `(x, y)` tuple, an empty `()`, a nested paren
370/// (`((x))`), a literal (`(1)`), or any compound expression (`(x::T)`, `(a+b)`)
371/// is not. A `TUPLE_EXPR` (a comma/empty paren) is always invalid.
372fn export_paren_is_symbol(item: &SyntaxNode) -> bool {
373 if item.kind() != SyntaxKind::PAREN_EXPR {
374 return false;
375 }
376 let mut significant = item.children_with_tokens().filter(|el| {
377 !matches!(
378 el.kind(),
379 SyntaxKind::WHITESPACE
380 | SyntaxKind::NEWLINE
381 | SyntaxKind::COMMENT
382 | SyntaxKind::BLOCK_COMMENT
383 | SyntaxKind::LPAREN
384 | SyntaxKind::RPAREN
385 )
386 });
387 let Some(first) = significant.next() else {
388 return false;
389 };
390 if significant.next().is_some() {
391 return false;
392 }
393 match first {
394 // A symbol-shaped node: a bare name, `var"…"`, or `$`-interpolation. A
395 // `LITERAL`, `TYPE_ANNOTATION`, nested `PAREN_EXPR`, etc. is not.
396 rowan::NodeOrToken::Node(n) => match n.kind() {
397 SyntaxKind::NAME | SyntaxKind::NONSTANDARD_IDENTIFIER | SyntaxKind::INTERPOLATION => {
398 true
399 }
400 // A parenthesized operator value (`(..)`, `(√)`, `(⊕)`) — which
401 // `parse_paren` wraps in an `OPERATOR_ATOM` — is a valid export
402 // symbol, but a dotted broadcast operator (`(.+)`, `(.≤)`) is not:
403 // JuliaSyntax error-wraps it (`export (.+)` ⇒ `(export (error (. +)))`).
404 SyntaxKind::OPERATOR_ATOM => !export_operator_atom_is_dotted(&n),
405 _ => false,
406 },
407 // The only bare token a `PAREN_EXPR` carries is the lone operator-value
408 // form (`(+)`, `(:)`), which `parse_paren` builds for `is_paren_value_op`.
409 rowan::NodeOrToken::Token(_) => true,
410 }
411}
412
413/// Whether a parenthesized `export`/`import` operator atom is a *dotted*
414/// (broadcast) operator (`.+`, `.≤`, `.&&`), which JuliaSyntax rejects as an
415/// export symbol. The range/splat operators `..`/`...` lead with a *doubled*
416/// dot and are not broadcasts, so they are valid; a lone `√`/`⊕` has no dot.
417fn export_operator_atom_is_dotted(atom: &SyntaxNode) -> bool {
418 atom.children_with_tokens()
419 .filter_map(|el| el.into_token())
420 .find(|t| {
421 !matches!(
422 t.kind(),
423 SyntaxKind::WHITESPACE
424 | SyntaxKind::NEWLINE
425 | SyntaxKind::COMMENT
426 | SyntaxKind::BLOCK_COMMENT
427 )
428 })
429 .is_some_and(|t| {
430 let bytes = t.text().as_bytes();
431 bytes.first() == Some(&b'.') && bytes.len() > 1 && bytes[1] != b'.'
432 })
433}
434
435/// Flag each `const` whose declaration is not the plain `=` assignment
436/// JuliaSyntax requires. A bare declaration (`const x`), a non-`=` assignment
437/// (`const x += 1`, `const x .= 1`), or a `global`/`local` declaration without an
438/// `=` (`const global x`) is invalid, so JuliaSyntax wraps the whole `const` in
439/// `(error …)` (`const x` ⇒ `(error (const x))`). A valid `const x = 1` — or a
440/// `global`/`local`-wrapped `=` (`const global x = 1`) — is left alone. The
441/// diagnostic anchors at the `const` keyword start.
442fn flag_invalid_const_decls(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
443 for node in cst
444 .descendants()
445 .filter(|n| n.kind() == SyntaxKind::CONST_STMT)
446 {
447 if !const_decl_is_assignment(&node) && !is_struct_const_field(&node) {
448 let pos = usize::from(node.text_range().start());
449 push_diagnostic(
450 diagnostics,
451 DiagnosticKind::ConstNotAssignment,
452 "expected assignment after `const`",
453 pos,
454 pos,
455 );
456 }
457 }
458}
459
460/// Flag each invalid `function`/`macro` signature. A bare name is a valid forward
461/// declaration only with a truly empty body; a direct parenthesized macro call or
462/// interpolation is an invalid anonymous-function parameter tuple. JuliaSyntax
463/// error-wraps either signature, and the diagnostic anchors at `SIGNATURE`.
464fn flag_invalid_function_signatures(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
465 for node in cst
466 .descendants()
467 .filter(|n| matches!(n.kind(), SyntaxKind::FUNCTION_DEF | SyntaxKind::MACRO_DEF))
468 {
469 let Some(sig) = node.children().find(|c| c.kind() == SyntaxKind::SIGNATURE) else {
470 continue;
471 };
472 let sig_is_bare_name = sig
473 .first_child()
474 .is_some_and(|inner| is_bare_signature_name(&inner));
475 let invalid_bare_name = sig_is_bare_name
476 && (!function_body_is_empty(&node) || invalid_forward_operator_name(&sig));
477 let invalid_parenthesized_macro = invalid_parenthesized_macro_signature(&sig);
478 if invalid_bare_name || invalid_parenthesized_macro {
479 let pos = usize::from(sig.text_range().start());
480 push_diagnostic(
481 diagnostics,
482 DiagnosticKind::InvalidFunctionSignature,
483 if invalid_bare_name {
484 "invalid function signature: expected a call, not a bare name"
485 } else {
486 "invalid function signature: macro call or interpolation cannot bind a parameter"
487 },
488 pos,
489 pos,
490 );
491 }
492 }
493}
494
495/// Whether a singleton anonymous-function parameter list is a macro call or an
496/// interpolation. These expression forms cannot bind a parameter, so JuliaSyntax
497/// wraps the whole tuple in an error. A comma, semicolon, or splat makes the
498/// corresponding tuple shape valid and therefore stays outside this recovery.
499fn invalid_parenthesized_macro_signature(signature: &SyntaxNode) -> bool {
500 let Some(tuple) = signature
501 .first_child()
502 .filter(|node| node.kind() == SyntaxKind::TUPLE_EXPR)
503 else {
504 return false;
505 };
506 if tuple.children().count() != 1
507 || tuple
508 .children_with_tokens()
509 .any(|element| element.kind() == SyntaxKind::COMMA)
510 {
511 return false;
512 }
513
514 let mut child = tuple.first_child();
515 while child
516 .as_ref()
517 .is_some_and(|node| node.kind() == SyntaxKind::PAREN_EXPR)
518 {
519 child = child.and_then(|node| node.first_child());
520 }
521 child.is_some_and(|node| {
522 matches!(
523 node.kind(),
524 SyntaxKind::MACRO_CALL | SyntaxKind::INTERPOLATION
525 )
526 })
527}
528
529/// JuliaSyntax 1.0 rejects syntactic, broadcast, and bare `&` operators as
530/// forward-declaration names even though ordinary operators remain valid.
531fn invalid_forward_operator_name(signature: &SyntaxNode) -> bool {
532 let Some(inner) = signature.first_child() else {
533 return false;
534 };
535 if inner.kind() == SyntaxKind::ERROR {
536 return true;
537 }
538 if inner.kind() != SyntaxKind::OPERATOR_ATOM {
539 return false;
540 }
541 inner.first_token().is_some_and(|token| {
542 let text = token.text();
543 text == "&" || (text.starts_with('.') && !text.starts_with(".."))
544 })
545}
546
547/// Whether a `function`/`macro` signature's first node is a *bare name* — the
548/// thing being declared rather than a call or other expression. Besides a plain
549/// or `var"…"` identifier and a `$`-interpolation, an operator name counts
550/// (`function + end`, `function .+ end`), including one Julia rejects as a name,
551/// which the parser leaves as an `ERROR`-wrapped operator atom (`function = end`
552/// ⇒ `(function (error =))`).
553pub(super) fn is_bare_signature_name(node: &SyntaxNode) -> bool {
554 match node.kind() {
555 SyntaxKind::NAME
556 | SyntaxKind::NONSTANDARD_IDENTIFIER
557 | SyntaxKind::INTERPOLATION
558 | SyntaxKind::OPERATOR_ATOM => true,
559 SyntaxKind::ERROR => node
560 .first_child()
561 .is_some_and(|inner| inner.kind() == SyntaxKind::OPERATOR_ATOM),
562 _ => false,
563 }
564}
565
566/// Flag each `catch` clause whose variable is not a plain identifier. The catch
567/// variable must be a bare identifier (`catch e`), a `$`-interpolation
568/// (`catch $e`), or a `var"…"` non-standard identifier (`catch var"e"`); any
569/// other expression (`catch e+3`, `catch e.f`, `catch f(e)`, `catch 3`) is
570/// invalid and JuliaSyntax wraps it in `(error …)` (`catch e+3` ⇒ `(catch
571/// (error (call-i e + 3)) …)`). The diagnostic anchors at the catch-variable
572/// node's start.
573fn flag_invalid_catch_vars(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
574 for node in cst
575 .descendants()
576 .filter(|n| n.kind() == SyntaxKind::CATCH_CLAUSE)
577 {
578 let Some(var) = node.children().find(|c| c.kind() != SyntaxKind::BLOCK) else {
579 continue;
580 };
581 if !matches!(
582 var.kind(),
583 SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::NONSTANDARD_IDENTIFIER
584 ) {
585 let pos = usize::from(var.text_range().start());
586 push_diagnostic(
587 diagnostics,
588 DiagnosticKind::CatchVarNotIdentifier,
589 "catch variable must be an identifier",
590 pos,
591 pos,
592 );
593 }
594 }
595}
596
597/// Whether a `function`/`macro`'s body is syntactically empty — no statement
598/// nodes and no `;` separator opening the block. A bare-name header keeps the
599/// forward-declaration form only while this holds; a `;` (`function f; end`) or
600/// any body statement marks the block as genuinely opened.
601fn function_body_is_empty(node: &SyntaxNode) -> bool {
602 match node.children().find(|c| c.kind() == SyntaxKind::BLOCK) {
603 Some(block) => {
604 block.first_child().is_none()
605 && !block
606 .children_with_tokens()
607 .any(|el| el.kind() == SyntaxKind::SEMICOLON)
608 }
609 None => true,
610 }
611}
612
613/// Whether `node` is a `const` field declaration directly inside a struct body
614/// (`struct A const a end` ⇒ `(const a)`, not wrapped). Such a bare `const` is
615/// valid; the exemption is narrow — a `const` nested inside an `if`/`begin`
616/// within the struct (its block's parent is not the `STRUCT_DEF`) is still an
617/// error.
618fn is_struct_const_field(node: &SyntaxNode) -> bool {
619 node.parent()
620 .filter(|b| b.kind() == SyntaxKind::BLOCK)
621 .and_then(|b| b.parent())
622 .is_some_and(|p| p.kind() == SyntaxKind::STRUCT_DEF)
623}
624
625/// Whether a `const`'s declaration is a plain `=` assignment. The body is the
626/// `const`'s first child node, unwrapping any `global`/`local` declaration; it is
627/// valid only when it is an `ASSIGNMENT_EXPR` headed by a plain `=` (not
628/// `.=`/`+=`/…).
629fn const_decl_is_assignment(node: &SyntaxNode) -> bool {
630 let mut body = node.first_child();
631 while let Some(n) = body {
632 match n.kind() {
633 SyntaxKind::GLOBAL_STMT | SyntaxKind::LOCAL_STMT => body = n.first_child(),
634 SyntaxKind::ASSIGNMENT_EXPR => {
635 return n
636 .children_with_tokens()
637 .filter_map(|el| el.into_token())
638 .any(|t| t.kind() == SyntaxKind::EQ);
639 }
640 _ => return false,
641 }
642 }
643 false
644}
645
646/// Whether `kind` is a closing delimiter (`)`, `]`, `}`), which when stray at
647/// statement start drives JuliaSyntax's leading-`(error)` recovery.
648fn is_close_delimiter_tok(kind: TokKind) -> bool {
649 matches!(kind, TokKind::RParen | TokKind::RBracket | TokKind::RBrace)
650}
651
652/// Whether `kind` is a middle/closing block keyword (`end`, `else`, `elseif`,
653/// `catch`, `finally`) — a token that only closes or continues an enclosing
654/// block. Where a statement is expected one of these is stray, so JuliaSyntax
655/// wraps it alone in `(error <kw>)` (`@doc x\nend` ⇒ `(macrocall @doc x) (error
656/// end)`, `end y z` ⇒ `(error end) (error-t y z)`).
657fn is_stray_block_keyword_tok(kind: TokKind) -> bool {
658 matches!(
659 kind,
660 TokKind::EndKw
661 | TokKind::ElseKw
662 | TokKind::ElseifKw
663 | TokKind::CatchKw
664 | TokKind::FinallyKw
665 )
666}
667
668/// Whether the logical line starting at `start` (up to the next newline or EOF)
669/// carries a top-level `;`. The stray-closer recovery only applies to the clean
670/// separator-less form; `;`-segment lines keep the loose-token fallback.
671fn rest_of_line_has_semicolon(tokens: &[Token], start: usize) -> bool {
672 tokens[start..]
673 .iter()
674 .take_while(|t| t.kind != TokKind::Newline)
675 .any(|t| t.kind == TokKind::Semicolon)
676}
677
678/// Whether an event carries significant (non-trivia) content: any node opener,
679/// or a non-trivia leaf token.
680fn is_significant_event(event: &Event, tokens: &[Token]) -> bool {
681 match event {
682 Event::Start(_) => true,
683 Event::Tok(idx) => !tokens[*idx].kind.is_trivia(),
684 Event::Finish => false,
685 }
686}
687
688/// Whether the first significant event in `tail` opens a node (a real statement
689/// subtree). A docstring's trailing leftover defers to `fold_docstrings` only
690/// when it begins with such a subtree (its documentable target); a leftover that
691/// opens with a bare junk token is recovered as trailing junk instead.
692fn leftover_starts_with_subtree(tail: &[Event], tokens: &[Token]) -> bool {
693 matches!(
694 tail.iter().find(|e| is_significant_event(e, tokens)),
695 Some(Event::Start(_))
696 )
697}
698
699/// Whether a statement's events open a bare (doc-eligible) `STRING_LITERAL` —
700/// the first inner token is not a `STRING_PREFIX`. Such a statement starts a
701/// potential docstring, so a trailing statement on the same logical line is left
702/// to `fold_docstrings` rather than wrapped as junk.
703pub(super) fn stmt_is_doc_string(events: &[Event], tokens: &[Token]) -> bool {
704 matches!(
705 events.first(),
706 Some(Event::Start(SyntaxKind::STRING_LITERAL))
707 ) && string_is_doc_eligible(&events[1..], tokens)
708}
709
710/// Whether a node of `kind` is a statement container in which the docstring fold
711/// applies (a string literal statement directly followed by another statement).
712fn is_doc_container(kind: SyntaxKind) -> bool {
713 matches!(kind, SyntaxKind::BLOCK | SyntaxKind::TOPLEVEL_SEMICOLON)
714}
715
716/// Whether a `STRING_LITERAL` subtree is a plain string (doc-eligible) rather
717/// than a prefixed string macro (`r"…"`, `b"…"`), which is a macro call in
718/// JuliaSyntax and never a docstring. Eligible iff its first token is not a
719/// `STRING_PREFIX`.
720fn string_is_doc_eligible(inner: &[Event], tokens: &[Token]) -> bool {
721 match inner.first() {
722 Some(Event::Tok(idx)) => syntax_kind_for(tokens[*idx].kind) != SyntaxKind::STRING_PREFIX,
723 _ => true,
724 }
725}
726
727/// Fold docstrings across the whole event stream. A bare unprefixed
728/// `STRING_LITERAL` statement immediately followed by another statement — at
729/// most one newline of intervening trivia, no `;` — folds into a `DOC` node
730/// `(doc str target)`, mirroring JuliaSyntax's `parse_docstring`. The pass
731/// descends into every subtree so nested blocks (function/module/begin bodies)
732/// fold too.
733///
734/// The fold only ever *inserts* a `Start(DOC)` before the string's `Start` and a
735/// matching `Finish` after the target's `Finish` — every other event keeps its
736/// order and its meaning. So rather than rebuilding the stream level by level,
737/// this marks the two insertion points per fold and splices them in with one
738/// final pass, which keeps the whole thing linear in the event count.
739fn fold_docstrings(events: &[Event], tokens: &[Token]) -> Vec<Event> {
740 let extents = subtree_extents(events);
741 let mut marks = DocMarks {
742 open: vec![false; events.len()],
743 close: vec![false; events.len()],
744 folds: 0,
745 };
746 mark_doc_folds(events, tokens, &extents, 0, events.len(), true, &mut marks);
747
748 if marks.folds == 0 {
749 return events.to_vec();
750 }
751
752 let mut out = Vec::with_capacity(events.len() + 2 * marks.folds);
753 for (i, event) in events.iter().enumerate() {
754 // A fold opens before a `Start` and closes after a `Finish`, so the two
755 // marks can never land on the same event and their order is unambiguous.
756 if marks.open[i] {
757 out.push(Event::Start(SyntaxKind::DOC));
758 }
759 out.push(event.clone());
760 if marks.close[i] {
761 out.push(Event::Finish);
762 }
763 }
764 out
765}
766
767/// Where each fold inserts its `DOC` wrapper: `open[i]` puts a `Start(DOC)`
768/// before event `i`, `close[i]` puts a `Finish` after it.
769struct DocMarks {
770 open: Vec<bool>,
771 close: Vec<bool>,
772 folds: usize,
773}
774
775/// For every `Start` in `events`, the index of its matching `Finish`; `usize::MAX`
776/// for any unmatched opener. One stack pass, so a later walk can skip a whole
777/// subtree in constant time instead of rescanning it for the matching close.
778fn subtree_extents(events: &[Event]) -> Vec<usize> {
779 let mut extents = vec![usize::MAX; events.len()];
780 let mut open: Vec<usize> = Vec::new();
781 for (i, event) in events.iter().enumerate() {
782 match event {
783 Event::Start(_) => open.push(i),
784 Event::Finish => {
785 if let Some(start) = open.pop() {
786 extents[start] = i;
787 }
788 }
789 Event::Tok(_) => {}
790 }
791 }
792 extents
793}
794
795/// Walk the direct children of the level spanning `start..end`, recording a fold
796/// for each docstring pair and descending into every subtree. `is_container` says
797/// whether a fold may happen at *this* level.
798fn mark_doc_folds(
799 events: &[Event],
800 tokens: &[Token],
801 extents: &[usize],
802 start: usize,
803 end: usize,
804 is_container: bool,
805 marks: &mut DocMarks,
806) {
807 let mut i = start;
808 while i < end {
809 let Event::Start(kind) = events[i] else {
810 i += 1;
811 continue;
812 };
813 // An unmatched opener runs to the end of this level.
814 let stop = extents[i].min(end);
815 mark_doc_folds(
816 events,
817 tokens,
818 extents,
819 i + 1,
820 stop,
821 is_doc_container(kind),
822 marks,
823 );
824
825 if is_container
826 && kind == SyntaxKind::STRING_LITERAL
827 && string_is_doc_eligible(&events[i + 1..stop], tokens)
828 && let Some(target) = doc_target(events, tokens, stop + 1, end)
829 {
830 let target_end = extents[target].min(end);
831 // The target is consumed by this fold: it is documented *by* the
832 // string, so it never goes on to document what follows it. Descend
833 // into it here, then resume past it.
834 mark_doc_folds(
835 events,
836 tokens,
837 extents,
838 target + 1,
839 target_end,
840 is_doc_container(as_start_kind(&events[target])),
841 marks,
842 );
843 marks.open[i] = true;
844 marks.close[target_end] = true;
845 marks.folds += 1;
846 i = target_end + 1;
847 continue;
848 }
849 i = stop + 1;
850 }
851}
852
853/// The kind of a `Start` event. Only ever called on an event already matched as
854/// one, so the fallback kind is unreachable.
855fn as_start_kind(event: &Event) -> SyntaxKind {
856 match event {
857 Event::Start(kind) => *kind,
858 _ => SyntaxKind::ERROR,
859 }
860}
861
862/// Scanning forward from `from` within a level ending at `end`, the index of the
863/// `Start` of the statement a preceding docstring documents: the next subtree
864/// child, reachable across at most one newline of trivia and no `;`. `None` if no
865/// eligible target follows.
866fn doc_target(events: &[Event], tokens: &[Token], from: usize, end: usize) -> Option<usize> {
867 let mut newlines = 0;
868 let mut j = from;
869 while j < end {
870 match events[j] {
871 // An error-recovery node is never a documentable target (`"doc"\n]`
872 // ⇒ `(string) (error) (error-t ✘)`, not a `(doc …)`); the string is
873 // a plain statement.
874 Event::Start(SyntaxKind::ERROR) => return None,
875 Event::Start(_) => return Some(j),
876 Event::Tok(t) => {
877 let kind = tokens[t].kind;
878 if kind == TokKind::Newline {
879 newlines += 1;
880 if newlines > 1 {
881 return None;
882 }
883 } else if !kind.is_trivia() {
884 return None;
885 }
886 j += 1;
887 }
888 // Only reachable on an unbalanced stream, where it ends the level.
889 Event::Finish => return None,
890 }
891 }
892 None
893}
894
895/// Round-trip the input through the parser: concatenating every token in the CST
896/// must reproduce the original text exactly (the losslessness invariant).
897pub fn reconstruct(text: &str) -> String {
898 parse(text)
899 .cst
900 .descendants_with_tokens()
901 .filter_map(|el| el.into_token())
902 .map(|tok| tok.text().to_string())
903 .collect()
904}
905
906/// Lexical shapes are independent of the edition; grammatical keyword
907/// classification happens in the parser context.
908pub(super) fn lex_for_options(text: &str, _options: ParseOptions) -> Vec<Token<'_>> {
909 lex(text)
910}
911
912#[cfg(test)]
913mod tests {
914 use super::*;
915
916 fn assert_lossless(input: &str) {
917 assert_eq!(reconstruct(input), input, "not lossless: {input:?}");
918 }
919
920 #[test]
921 fn lossless_corpus() {
922 for input in [
923 "",
924 "x = 1\n",
925 "y = a + b * c\n",
926 "f(a, b, c)\n",
927 "v[1]\n",
928 "a[end]\n",
929 "a[end - 1]\n",
930 "a[2:end]\n",
931 "m[end, end]\n",
932 "z = (a + b) / 2\n",
933 "function g(x)\n x ^ 2\nend\n",
934 "if a >= b\n a\nelseif c\n c\nelse\n b\nend\n",
935 "begin\n a\n b\nend\n",
936 "map(xs) do x\n x + 1\nend\n",
937 "open(\"f\") do\n read()\nend\n",
938 "# a comment\nx = 1 # trailing\n",
939 "#= block =#\nq = 2\n",
940 "obj.field\n",
941 "x::Int\n",
942 "Vector{T}\n",
943 "foo(x::T) where {T<:Number} = x\n",
944 "f(a, b; c=1, d=2)\n",
945 "g(args...; kwargs...)\n",
946 "y = a .+ b .* c\n",
947 "z = .-x\n",
948 "a .= f.(b, c)\n",
949 "t = (a, b)\n",
950 "u = (a,)\n",
951 "e = ()\n",
952 "nt = (x = 1, y = 2)\n",
953 "y = a ? b : c\n",
954 "z = a == b ? x + 1 : x - 1\n",
955 "w = a ? b : c ? d : e\n",
956 "v = [1, 2, 3]\n",
957 "e = []\n",
958 "m = [1 2; 3 4]\n",
959 "s = [1 +2]\n",
960 "c = [1; 2; 3]\n",
961 "q = [x for x in xs]\n",
962 "r = [x^2 for x in 1:10 if x > 2]\n",
963 "g = (x for x in xs)\n",
964 "a; b\n",
965 "a;;;b;;\n",
966 ";a\n",
967 "@assert nl > 1 # trailing comment\nnx = 2\n",
968 "@foo x # c\n",
969 "x = @bind 🐶 html\"<input>\"\n",
970 "begin @m 10 end\n",
971 ] {
972 assert_lossless(input);
973 }
974 }
975
976 #[test]
977 fn groups_top_level_semicolon_line() {
978 let out = parse("a; b\nc\n");
979 let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
980 use crate::syntax::SyntaxKind::{NAME, TOPLEVEL_SEMICOLON};
981 assert_eq!(kinds, vec![TOPLEVEL_SEMICOLON, NAME]);
982 }
983
984 #[test]
985 fn builds_expected_top_level_shape() {
986 let out = parse("x = 1 + 2\n");
987 let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
988 assert_eq!(kinds, vec![crate::syntax::SyntaxKind::ASSIGNMENT_EXPR]);
989 assert!(out.diagnostics.is_empty());
990 }
991
992 #[test]
993 fn overflowing_decimal_floats_are_parse_errors() {
994 let out = parse("x=1e400\ny=1f400\nz=1e-4000\n");
995 assert_eq!(
996 out.diagnostics
997 .iter()
998 .map(|diagnostic| (
999 diagnostic.start,
1000 diagnostic.end,
1001 diagnostic.message.as_str()
1002 ))
1003 .collect::<Vec<_>>(),
1004 vec![
1005 (2, 7, "overflow in floating point literal"),
1006 (10, 15, "overflow in floating point literal"),
1007 ]
1008 );
1009 }
1010}