fatou_parser/parser/core.rs
1use crate::parser::events::Event;
2use crate::parser::expr::parse_stmt;
3use crate::parser::lexer::{TokKind, Token, lex};
4use crate::parser::tree_builder::{build_tree, syntax_kind_for};
5use crate::syntax::{SyntaxKind, SyntaxNode};
6
7pub use crate::parser::diagnostics::ParseDiagnostic;
8use crate::parser::diagnostics::{DiagnosticKind, push_diagnostic};
9
10/// The result of a parse: the lossless CST plus any diagnostics.
11#[derive(Debug, Clone)]
12pub struct ParseOutput {
13 pub cst: SyntaxNode,
14 pub diagnostics: Vec<ParseDiagnostic>,
15}
16
17/// Parse `text` into a lossless `rowan` CST. The drive loop works one logical
18/// line at a time: leading trivia (including newlines) is emitted directly at
19/// the root, then the line's statements are parsed with the Pratt parser. A line
20/// that carries a top-level `;` groups its statements into a `TOPLEVEL_SEMICOLON`
21/// node (mirroring JuliaSyntax's `toplevel-;`); a plain line stays bare. An
22/// unparseable token is consumed as one error token so the loop always makes
23/// progress.
24pub fn parse(text: &str) -> ParseOutput {
25 let tokens = lex(text);
26 let mut diagnostics = Vec::new();
27 let mut events = Vec::new();
28
29 let mut i = 0usize;
30 while i < tokens.len() {
31 // Leading trivia and blank/newline runs belong directly at the root.
32 if tokens[i].kind.is_trivia() {
33 events.push(Event::Tok(i));
34 i += 1;
35 continue;
36 }
37
38 // Collect a single logical line (up to the next newline or EOF),
39 // tracking whether it carries a `;` separator. `leftover_mark` records
40 // the event offset right after the line's first complete statement so a
41 // trailing junk run on a separator-less line can be wrapped in an
42 // `ERROR` node and flagged with a `TrailingJunk` diagnostic (projected
43 // `(error-t …)`: `x y` ⇒ `x (error-t y)`).
44 let mut line = Vec::new();
45 let mut has_semicolon = false;
46 let mut leftover_mark: Option<usize> = None;
47 let mut first_is_doc_string = false;
48 // A doc-eligible string is only a docstring when a *real* statement
49 // follows it. If the leftover after the string can't start a statement
50 // (a stray closer/keyword: `"doc" ]`), the string is a plain statement
51 // and the remainder is junk — un-gate the raw trailing-junk collection.
52 let mut doc_no_target = false;
53 // A separator-less line: after the first complete statement, JuliaSyntax
54 // bumps the remainder as flat error tokens rather than re-parsing it. A
55 // line that carries a `;` keeps the per-segment behavior (deferred), so
56 // flat-junk collection is gated off when one is present.
57 let line_has_semicolon = rest_of_line_has_semicolon(&tokens, i);
58 while i < tokens.len() {
59 match tokens[i].kind {
60 TokKind::Newline => break,
61 TokKind::Semicolon => {
62 has_semicolon = true;
63 line.push(Event::Tok(i));
64 i += 1;
65 }
66 k if k.is_trivia() => {
67 line.push(Event::Tok(i));
68 i += 1;
69 }
70 _ => {
71 if !line_has_semicolon
72 && leftover_mark.is_some()
73 && (!first_is_doc_string || doc_no_target)
74 {
75 // Trailing junk after the first statement: collect raw
76 // (no structural re-parse) so the wrapping below puts the
77 // whole run in one ERROR node and the projector renders
78 // delimiters, commas, and `@` as `✘` (`x y, z` ⇒
79 // `x (error-t y ✘ z)`, `x@y` ⇒ `x (error-t ✘ y)`).
80 line.push(Event::Tok(i));
81 i += 1;
82 } else if let Some(expr) = parse_stmt(&tokens, i, &mut diagnostics) {
83 if leftover_mark.is_none() {
84 first_is_doc_string = stmt_is_doc_string(&expr.events, &tokens);
85 }
86 line.extend(expr.events);
87 i = expr.end;
88 leftover_mark.get_or_insert(line.len());
89 } else if leftover_mark.is_none()
90 && is_close_delimiter_tok(tokens[i].kind)
91 && !rest_of_line_has_semicolon(&tokens, i)
92 {
93 // A stray *closing* delimiter at statement start (no
94 // preceding statement) is JuliaSyntax's leading empty
95 // `(error)` plus an `(error-t ✘ …)` that swallows the
96 // rest of the line: `)` ⇒ `(error) (error-t ✘)`,
97 // `) x` ⇒ `(error) (error-t ✘ x)`, `)))` ⇒ `… ✘ ✘ ✘`.
98 // The `;`-segment forms emit a subtler double marker, so
99 // they are left to the loose-token fallback below.
100 line.push(Event::Start(SyntaxKind::ERROR));
101 line.push(Event::Finish);
102 let run_start = tokens[i].start;
103 line.push(Event::Start(SyntaxKind::ERROR));
104 while i < tokens.len() && tokens[i].kind != TokKind::Newline {
105 line.push(Event::Tok(i));
106 i += 1;
107 }
108 line.push(Event::Finish);
109 push_diagnostic(
110 &mut diagnostics,
111 DiagnosticKind::StrayCloser,
112 "stray closing delimiter",
113 run_start,
114 tokens[i - 1].end,
115 );
116 } else if leftover_mark.is_none() && is_stray_block_keyword_tok(tokens[i].kind)
117 {
118 // A middle/closing block keyword (`end`, `else`,
119 // `elseif`, `catch`, `finally`) where a statement is
120 // expected is not a block opener; JuliaSyntax wraps it
121 // alone in `(error <kw>)` and bumps the rest of the line
122 // as a separate trailing-junk run (`end y z` ⇒
123 // `(error end) (error-t y z)`). Emit the wrapped keyword
124 // as the line's first "statement" and set `leftover_mark`
125 // so the existing trailing-junk machinery handles the
126 // remainder.
127 line.push(Event::Start(SyntaxKind::ERROR));
128 line.push(Event::Tok(i));
129 line.push(Event::Finish);
130 push_diagnostic(
131 &mut diagnostics,
132 DiagnosticKind::StrayKeyword,
133 "unexpected block keyword",
134 tokens[i].start,
135 tokens[i].end,
136 );
137 i += 1;
138 leftover_mark = Some(line.len());
139 } else {
140 // A leftover token that can't start a statement. After a
141 // doc-eligible string this means the string has no
142 // documentable target, so the rest of the line is junk.
143 if first_is_doc_string && leftover_mark.is_some() {
144 doc_no_target = true;
145 }
146 line.push(Event::Tok(i));
147 i += 1;
148 }
149 }
150 }
151 }
152
153 if has_semicolon {
154 events.push(Event::Start(SyntaxKind::TOPLEVEL_SEMICOLON));
155 events.extend(line);
156 events.push(Event::Finish);
157 } else if let Some(mark) = leftover_mark.filter(|&m| {
158 // Junk after the first statement on a separator-less line — but a
159 // bare docstring (`"a"\nfoo`) owns its trailing statement, so leave
160 // that to `fold_docstrings`. A docstring defers only when its
161 // leftover actually begins with a documentable statement subtree
162 // (`"doc" foo`); a leftover that opens with junk (`"doc" ]`) is
163 // recovered here so the stray closer becomes `(error-t ✘)`.
164 let tail = &line[m..];
165 let defer_to_doc = first_is_doc_string && leftover_starts_with_subtree(tail, &tokens);
166 !defer_to_doc && tail.iter().any(|e| is_significant_event(e, &tokens))
167 }) {
168 events.extend(line[..mark].iter().cloned());
169 let tail = &line[mark..];
170 // Leading trivia stays outside the error node; everything from the
171 // first significant leftover token onward is the recovered run.
172 let lead = tail
173 .iter()
174 .take_while(|e| !is_significant_event(e, &tokens))
175 .count();
176 events.extend(tail[..lead].iter().cloned());
177 let first_junk = tail[lead..].iter().find_map(|e| match e {
178 Event::Tok(idx) => Some(*idx),
179 _ => None,
180 });
181 events.push(Event::Start(SyntaxKind::ERROR));
182 events.extend(tail[lead..].iter().cloned());
183 events.push(Event::Finish);
184 if let Some(idx) = first_junk {
185 push_diagnostic(
186 &mut diagnostics,
187 DiagnosticKind::TrailingJunk,
188 "trailing tokens after statement",
189 tokens[idx].start,
190 tokens[idx].end,
191 );
192 }
193 } else {
194 events.extend(line);
195 }
196 }
197
198 finish_parse(&tokens, &events, diagnostics)
199}
200
201/// Reparse statements using the same body driver as their enclosing construct.
202/// Keeping the owner in the temporary tree lets the flag passes recognize
203/// context-sensitive forms such as a struct's bare `const` fields.
204pub(super) fn parse_block_fragment(text: &str, owner: SyntaxKind) -> Option<ParseOutput> {
205 let tokens = lex(text);
206 let mut diagnostics = Vec::new();
207 let mut events = vec![Event::Start(owner)];
208 let end = super::structural::run_block_fragment(&tokens, &mut events, owner, &mut diagnostics)?;
209 if end != tokens.len() {
210 return None;
211 }
212 events.push(Event::Finish);
213 let mut output = finish_parse(&tokens, &events, diagnostics);
214 output.cst = output.cst.first_child()?.first_child()?;
215 Some(output)
216}
217
218fn finish_parse(
219 tokens: &[Token],
220 events: &[Event],
221 mut diagnostics: Vec<ParseDiagnostic>,
222) -> ParseOutput {
223 let events = fold_docstrings(events, tokens);
224
225 let cst = build_tree(tokens, &events);
226 // The post-build flag passes. Each walks the finished CST for a shape
227 // JuliaSyntax error-wraps and records a *zero-width point* diagnostic at the
228 // offending node's start; the projector reconstructs the error wrapper from
229 // that anchor, so the CST itself stays faithful to the source.
230 flag_invalid_const_decls(&cst, &mut diagnostics);
231 flag_invalid_function_signatures(&cst, &mut diagnostics);
232 flag_invalid_catch_vars(&cst, &mut diagnostics);
233 flag_invalid_export_items(&cst, &mut diagnostics);
234 flag_numeric_overflow(&cst, &mut diagnostics);
235 ParseOutput { cst, diagnostics }
236}
237
238/// Flag decimal floats that Julia rejects instead of accepting as infinity.
239/// Rust's parsers return an infinite value for these spellings, which gives us
240/// the same range check without evaluating Julia code.
241fn flag_numeric_overflow(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
242 for literal in cst
243 .descendants()
244 .filter(|node| node.kind() == SyntaxKind::LITERAL)
245 {
246 let Some(number) = literal
247 .children_with_tokens()
248 .filter_map(|element| element.into_token())
249 .find(|token| matches!(token.kind(), SyntaxKind::FLOAT | SyntaxKind::FLOAT32))
250 else {
251 continue;
252 };
253 if !decimal_float_overflows(number.text(), number.kind()) {
254 continue;
255 }
256 let range = literal.text_range();
257 push_diagnostic(
258 diagnostics,
259 DiagnosticKind::NumericOverflow,
260 "overflow in floating point literal",
261 usize::from(range.start()),
262 usize::from(range.end()),
263 );
264 }
265}
266
267fn decimal_float_overflows(text: &str, kind: SyntaxKind) -> bool {
268 if text.contains("0x") || text.contains("0X") {
269 return false;
270 }
271 let mut parseable = text.replace('_', "");
272 if kind == SyntaxKind::FLOAT32
273 && let Some(marker) = parseable.find(['f', 'F'])
274 {
275 parseable.replace_range(marker..=marker, "e");
276 }
277 match kind {
278 SyntaxKind::FLOAT32 => parseable
279 .parse::<f32>()
280 .is_ok_and(|value| !value.is_finite()),
281 SyntaxKind::FLOAT => parseable
282 .parse::<f64>()
283 .is_ok_and(|value| !value.is_finite()),
284 _ => false,
285 }
286}
287
288/// Flag each parenthesized `export` item that is not a single symbol. An export
289/// item may be parenthesized, but only around a single identifier, operator,
290/// `var"…"`, or `$`-interpolation (`export (x)`/`export (+)`/`export ($a)`),
291/// which JuliaSyntax unwraps to the bare symbol. Any other parenthesized form
292/// (`export (x::T)`, `export (x, y)`, `export ()`, `export ((x))`) is invalid and
293/// JuliaSyntax error-wraps the parsed expression (`(export (error (::-i x T)))`).
294fn flag_invalid_export_items(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
295 for stmt in cst
296 .descendants()
297 .filter(|n| n.kind() == SyntaxKind::EXPORT_STMT)
298 {
299 for item in stmt
300 .children()
301 .filter(|c| matches!(c.kind(), SyntaxKind::PAREN_EXPR | SyntaxKind::TUPLE_EXPR))
302 {
303 if !export_paren_is_symbol(&item) {
304 let pos = usize::from(item.text_range().start());
305 push_diagnostic(
306 diagnostics,
307 DiagnosticKind::InvalidExportItem,
308 "invalid `export` item: expected a single symbol",
309 pos,
310 pos,
311 );
312 }
313 }
314 }
315}
316
317/// Whether a parenthesized `export` item wraps a single symbol that JuliaSyntax
318/// unwraps: a lone identifier (`(x)`), `var"…"` (`(var"x")`), `$`-interpolation
319/// (`($a)`), or operator (`(+)`). A `(x, y)` tuple, an empty `()`, a nested paren
320/// (`((x))`), a literal (`(1)`), or any compound expression (`(x::T)`, `(a+b)`)
321/// is not. A `TUPLE_EXPR` (a comma/empty paren) is always invalid.
322fn export_paren_is_symbol(item: &SyntaxNode) -> bool {
323 if item.kind() != SyntaxKind::PAREN_EXPR {
324 return false;
325 }
326 let mut significant = item.children_with_tokens().filter(|el| {
327 !matches!(
328 el.kind(),
329 SyntaxKind::WHITESPACE
330 | SyntaxKind::NEWLINE
331 | SyntaxKind::COMMENT
332 | SyntaxKind::BLOCK_COMMENT
333 | SyntaxKind::LPAREN
334 | SyntaxKind::RPAREN
335 )
336 });
337 let Some(first) = significant.next() else {
338 return false;
339 };
340 if significant.next().is_some() {
341 return false;
342 }
343 match first {
344 // A symbol-shaped node: a bare name, `var"…"`, or `$`-interpolation. A
345 // `LITERAL`, `TYPE_ANNOTATION`, nested `PAREN_EXPR`, etc. is not.
346 rowan::NodeOrToken::Node(n) => match n.kind() {
347 SyntaxKind::NAME | SyntaxKind::NONSTANDARD_IDENTIFIER | SyntaxKind::INTERPOLATION => {
348 true
349 }
350 // A parenthesized operator value (`(..)`, `(√)`, `(⊕)`) — which
351 // `parse_paren` wraps in an `OPERATOR_ATOM` — is a valid export
352 // symbol, but a dotted broadcast operator (`(.+)`, `(.≤)`) is not:
353 // JuliaSyntax error-wraps it (`export (.+)` ⇒ `(export (error (. +)))`).
354 SyntaxKind::OPERATOR_ATOM => !export_operator_atom_is_dotted(&n),
355 _ => false,
356 },
357 // The only bare token a `PAREN_EXPR` carries is the lone operator-value
358 // form (`(+)`, `(:)`), which `parse_paren` builds for `is_paren_value_op`.
359 rowan::NodeOrToken::Token(_) => true,
360 }
361}
362
363/// Whether a parenthesized `export`/`import` operator atom is a *dotted*
364/// (broadcast) operator (`.+`, `.≤`, `.&&`), which JuliaSyntax rejects as an
365/// export symbol. The range/splat operators `..`/`...` lead with a *doubled*
366/// dot and are not broadcasts, so they are valid; a lone `√`/`⊕` has no dot.
367fn export_operator_atom_is_dotted(atom: &SyntaxNode) -> bool {
368 atom.children_with_tokens()
369 .filter_map(|el| el.into_token())
370 .find(|t| {
371 !matches!(
372 t.kind(),
373 SyntaxKind::WHITESPACE
374 | SyntaxKind::NEWLINE
375 | SyntaxKind::COMMENT
376 | SyntaxKind::BLOCK_COMMENT
377 )
378 })
379 .is_some_and(|t| {
380 let bytes = t.text().as_bytes();
381 bytes.first() == Some(&b'.') && bytes.len() > 1 && bytes[1] != b'.'
382 })
383}
384
385/// Flag each `const` whose declaration is not the plain `=` assignment
386/// JuliaSyntax requires. A bare declaration (`const x`), a non-`=` assignment
387/// (`const x += 1`, `const x .= 1`), or a `global`/`local` declaration without an
388/// `=` (`const global x`) is invalid, so JuliaSyntax wraps the whole `const` in
389/// `(error …)` (`const x` ⇒ `(error (const x))`). A valid `const x = 1` — or a
390/// `global`/`local`-wrapped `=` (`const global x = 1`) — is left alone. The
391/// diagnostic anchors at the `const` keyword start.
392fn flag_invalid_const_decls(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
393 for node in cst
394 .descendants()
395 .filter(|n| n.kind() == SyntaxKind::CONST_STMT)
396 {
397 if !const_decl_is_assignment(&node) && !is_struct_const_field(&node) {
398 let pos = usize::from(node.text_range().start());
399 push_diagnostic(
400 diagnostics,
401 DiagnosticKind::ConstNotAssignment,
402 "expected assignment after `const`",
403 pos,
404 pos,
405 );
406 }
407 }
408}
409
410/// Flag each invalid `function`/`macro` signature. A bare name is a valid forward
411/// declaration only with a truly empty body; a direct parenthesized macro call or
412/// interpolation is an invalid anonymous-function parameter tuple. JuliaSyntax
413/// error-wraps either signature, and the diagnostic anchors at `SIGNATURE`.
414fn flag_invalid_function_signatures(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
415 for node in cst
416 .descendants()
417 .filter(|n| matches!(n.kind(), SyntaxKind::FUNCTION_DEF | SyntaxKind::MACRO_DEF))
418 {
419 let Some(sig) = node.children().find(|c| c.kind() == SyntaxKind::SIGNATURE) else {
420 continue;
421 };
422 let sig_is_bare_name = sig
423 .first_child()
424 .is_some_and(|inner| is_bare_signature_name(&inner));
425 let invalid_bare_name = sig_is_bare_name
426 && (!function_body_is_empty(&node) || invalid_forward_operator_name(&sig));
427 let invalid_parenthesized_macro = invalid_parenthesized_macro_signature(&sig);
428 if invalid_bare_name || invalid_parenthesized_macro {
429 let pos = usize::from(sig.text_range().start());
430 push_diagnostic(
431 diagnostics,
432 DiagnosticKind::InvalidFunctionSignature,
433 if invalid_bare_name {
434 "invalid function signature: expected a call, not a bare name"
435 } else {
436 "invalid function signature: macro call or interpolation cannot bind a parameter"
437 },
438 pos,
439 pos,
440 );
441 }
442 }
443}
444
445/// Whether a singleton anonymous-function parameter list is a macro call or an
446/// interpolation. These expression forms cannot bind a parameter, so JuliaSyntax
447/// wraps the whole tuple in an error. A comma, semicolon, or splat makes the
448/// corresponding tuple shape valid and therefore stays outside this recovery.
449fn invalid_parenthesized_macro_signature(signature: &SyntaxNode) -> bool {
450 let Some(tuple) = signature
451 .first_child()
452 .filter(|node| node.kind() == SyntaxKind::TUPLE_EXPR)
453 else {
454 return false;
455 };
456 if tuple.children().count() != 1
457 || tuple
458 .children_with_tokens()
459 .any(|element| element.kind() == SyntaxKind::COMMA)
460 {
461 return false;
462 }
463
464 let mut child = tuple.first_child();
465 while child
466 .as_ref()
467 .is_some_and(|node| node.kind() == SyntaxKind::PAREN_EXPR)
468 {
469 child = child.and_then(|node| node.first_child());
470 }
471 child.is_some_and(|node| {
472 matches!(
473 node.kind(),
474 SyntaxKind::MACRO_CALL | SyntaxKind::INTERPOLATION
475 )
476 })
477}
478
479/// JuliaSyntax 1.0 rejects syntactic, broadcast, and bare `&` operators as
480/// forward-declaration names even though ordinary operators remain valid.
481fn invalid_forward_operator_name(signature: &SyntaxNode) -> bool {
482 let Some(inner) = signature.first_child() else {
483 return false;
484 };
485 if inner.kind() == SyntaxKind::ERROR {
486 return true;
487 }
488 if inner.kind() != SyntaxKind::OPERATOR_ATOM {
489 return false;
490 }
491 inner.first_token().is_some_and(|token| {
492 let text = token.text();
493 text == "&" || (text.starts_with('.') && !text.starts_with(".."))
494 })
495}
496
497/// Whether a `function`/`macro` signature's first node is a *bare name* — the
498/// thing being declared rather than a call or other expression. Besides a plain
499/// or `var"…"` identifier and a `$`-interpolation, an operator name counts
500/// (`function + end`, `function .+ end`), including one Julia rejects as a name,
501/// which the parser leaves as an `ERROR`-wrapped operator atom (`function = end`
502/// ⇒ `(function (error =))`).
503pub(super) fn is_bare_signature_name(node: &SyntaxNode) -> bool {
504 match node.kind() {
505 SyntaxKind::NAME
506 | SyntaxKind::NONSTANDARD_IDENTIFIER
507 | SyntaxKind::INTERPOLATION
508 | SyntaxKind::OPERATOR_ATOM => true,
509 SyntaxKind::ERROR => node
510 .first_child()
511 .is_some_and(|inner| inner.kind() == SyntaxKind::OPERATOR_ATOM),
512 _ => false,
513 }
514}
515
516/// Flag each `catch` clause whose variable is not a plain identifier. The catch
517/// variable must be a bare identifier (`catch e`), a `$`-interpolation
518/// (`catch $e`), or a `var"…"` non-standard identifier (`catch var"e"`); any
519/// other expression (`catch e+3`, `catch e.f`, `catch f(e)`, `catch 3`) is
520/// invalid and JuliaSyntax wraps it in `(error …)` (`catch e+3` ⇒ `(catch
521/// (error (call-i e + 3)) …)`). The diagnostic anchors at the catch-variable
522/// node's start.
523fn flag_invalid_catch_vars(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
524 for node in cst
525 .descendants()
526 .filter(|n| n.kind() == SyntaxKind::CATCH_CLAUSE)
527 {
528 let Some(var) = node.children().find(|c| c.kind() != SyntaxKind::BLOCK) else {
529 continue;
530 };
531 if !matches!(
532 var.kind(),
533 SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::NONSTANDARD_IDENTIFIER
534 ) {
535 let pos = usize::from(var.text_range().start());
536 push_diagnostic(
537 diagnostics,
538 DiagnosticKind::CatchVarNotIdentifier,
539 "catch variable must be an identifier",
540 pos,
541 pos,
542 );
543 }
544 }
545}
546
547/// Whether a `function`/`macro`'s body is syntactically empty — no statement
548/// nodes and no `;` separator opening the block. A bare-name header keeps the
549/// forward-declaration form only while this holds; a `;` (`function f; end`) or
550/// any body statement marks the block as genuinely opened.
551fn function_body_is_empty(node: &SyntaxNode) -> bool {
552 match node.children().find(|c| c.kind() == SyntaxKind::BLOCK) {
553 Some(block) => {
554 block.first_child().is_none()
555 && !block
556 .children_with_tokens()
557 .any(|el| el.kind() == SyntaxKind::SEMICOLON)
558 }
559 None => true,
560 }
561}
562
563/// Whether `node` is a `const` field declaration directly inside a struct body
564/// (`struct A const a end` ⇒ `(const a)`, not wrapped). Such a bare `const` is
565/// valid; the exemption is narrow — a `const` nested inside an `if`/`begin`
566/// within the struct (its block's parent is not the `STRUCT_DEF`) is still an
567/// error.
568fn is_struct_const_field(node: &SyntaxNode) -> bool {
569 node.parent()
570 .filter(|b| b.kind() == SyntaxKind::BLOCK)
571 .and_then(|b| b.parent())
572 .is_some_and(|p| p.kind() == SyntaxKind::STRUCT_DEF)
573}
574
575/// Whether a `const`'s declaration is a plain `=` assignment. The body is the
576/// `const`'s first child node, unwrapping any `global`/`local` declaration; it is
577/// valid only when it is an `ASSIGNMENT_EXPR` headed by a plain `=` (not
578/// `.=`/`+=`/…).
579fn const_decl_is_assignment(node: &SyntaxNode) -> bool {
580 let mut body = node.first_child();
581 while let Some(n) = body {
582 match n.kind() {
583 SyntaxKind::GLOBAL_STMT | SyntaxKind::LOCAL_STMT => body = n.first_child(),
584 SyntaxKind::ASSIGNMENT_EXPR => {
585 return n
586 .children_with_tokens()
587 .filter_map(|el| el.into_token())
588 .any(|t| t.kind() == SyntaxKind::EQ);
589 }
590 _ => return false,
591 }
592 }
593 false
594}
595
596/// Whether `kind` is a closing delimiter (`)`, `]`, `}`), which when stray at
597/// statement start drives JuliaSyntax's leading-`(error)` recovery.
598fn is_close_delimiter_tok(kind: TokKind) -> bool {
599 matches!(kind, TokKind::RParen | TokKind::RBracket | TokKind::RBrace)
600}
601
602/// Whether `kind` is a middle/closing block keyword (`end`, `else`, `elseif`,
603/// `catch`, `finally`) — a token that only closes or continues an enclosing
604/// block. Where a statement is expected one of these is stray, so JuliaSyntax
605/// wraps it alone in `(error <kw>)` (`@doc x\nend` ⇒ `(macrocall @doc x) (error
606/// end)`, `end y z` ⇒ `(error end) (error-t y z)`).
607fn is_stray_block_keyword_tok(kind: TokKind) -> bool {
608 matches!(
609 kind,
610 TokKind::EndKw
611 | TokKind::ElseKw
612 | TokKind::ElseifKw
613 | TokKind::CatchKw
614 | TokKind::FinallyKw
615 )
616}
617
618/// Whether the logical line starting at `start` (up to the next newline or EOF)
619/// carries a top-level `;`. The stray-closer recovery only applies to the clean
620/// separator-less form; `;`-segment lines keep the loose-token fallback.
621fn rest_of_line_has_semicolon(tokens: &[Token], start: usize) -> bool {
622 tokens[start..]
623 .iter()
624 .take_while(|t| t.kind != TokKind::Newline)
625 .any(|t| t.kind == TokKind::Semicolon)
626}
627
628/// Whether an event carries significant (non-trivia) content: any node opener,
629/// or a non-trivia leaf token.
630fn is_significant_event(event: &Event, tokens: &[Token]) -> bool {
631 match event {
632 Event::Start(_) => true,
633 Event::Tok(idx) => !tokens[*idx].kind.is_trivia(),
634 Event::Finish => false,
635 }
636}
637
638/// Whether the first significant event in `tail` opens a node (a real statement
639/// subtree). A docstring's trailing leftover defers to `fold_docstrings` only
640/// when it begins with such a subtree (its documentable target); a leftover that
641/// opens with a bare junk token is recovered as trailing junk instead.
642fn leftover_starts_with_subtree(tail: &[Event], tokens: &[Token]) -> bool {
643 matches!(
644 tail.iter().find(|e| is_significant_event(e, tokens)),
645 Some(Event::Start(_))
646 )
647}
648
649/// Whether a statement's events open a bare (doc-eligible) `STRING_LITERAL` —
650/// the first inner token is not a `STRING_PREFIX`. Such a statement starts a
651/// potential docstring, so a trailing statement on the same logical line is left
652/// to `fold_docstrings` rather than wrapped as junk.
653pub(super) fn stmt_is_doc_string(events: &[Event], tokens: &[Token]) -> bool {
654 matches!(
655 events.first(),
656 Some(Event::Start(SyntaxKind::STRING_LITERAL))
657 ) && string_is_doc_eligible(&events[1..], tokens)
658}
659
660/// Whether a node of `kind` is a statement container in which the docstring fold
661/// applies (a string literal statement directly followed by another statement).
662fn is_doc_container(kind: SyntaxKind) -> bool {
663 matches!(kind, SyntaxKind::BLOCK | SyntaxKind::TOPLEVEL_SEMICOLON)
664}
665
666/// Whether a `STRING_LITERAL` subtree is a plain string (doc-eligible) rather
667/// than a prefixed string macro (`r"…"`, `b"…"`), which is a macro call in
668/// JuliaSyntax and never a docstring. Eligible iff its first token is not a
669/// `STRING_PREFIX`.
670fn string_is_doc_eligible(inner: &[Event], tokens: &[Token]) -> bool {
671 match inner.first() {
672 Some(Event::Tok(idx)) => syntax_kind_for(tokens[*idx].kind) != SyntaxKind::STRING_PREFIX,
673 _ => true,
674 }
675}
676
677/// Fold docstrings across the whole event stream. A bare unprefixed
678/// `STRING_LITERAL` statement immediately followed by another statement — at
679/// most one newline of intervening trivia, no `;` — folds into a `DOC` node
680/// `(doc str target)`, mirroring JuliaSyntax's `parse_docstring`. The pass
681/// descends into every subtree so nested blocks (function/module/begin bodies)
682/// fold too.
683///
684/// The fold only ever *inserts* a `Start(DOC)` before the string's `Start` and a
685/// matching `Finish` after the target's `Finish` — every other event keeps its
686/// order and its meaning. So rather than rebuilding the stream level by level,
687/// this marks the two insertion points per fold and splices them in with one
688/// final pass, which keeps the whole thing linear in the event count.
689fn fold_docstrings(events: &[Event], tokens: &[Token]) -> Vec<Event> {
690 let extents = subtree_extents(events);
691 let mut marks = DocMarks {
692 open: vec![false; events.len()],
693 close: vec![false; events.len()],
694 folds: 0,
695 };
696 mark_doc_folds(events, tokens, &extents, 0, events.len(), true, &mut marks);
697
698 if marks.folds == 0 {
699 return events.to_vec();
700 }
701
702 let mut out = Vec::with_capacity(events.len() + 2 * marks.folds);
703 for (i, event) in events.iter().enumerate() {
704 // A fold opens before a `Start` and closes after a `Finish`, so the two
705 // marks can never land on the same event and their order is unambiguous.
706 if marks.open[i] {
707 out.push(Event::Start(SyntaxKind::DOC));
708 }
709 out.push(event.clone());
710 if marks.close[i] {
711 out.push(Event::Finish);
712 }
713 }
714 out
715}
716
717/// Where each fold inserts its `DOC` wrapper: `open[i]` puts a `Start(DOC)`
718/// before event `i`, `close[i]` puts a `Finish` after it.
719struct DocMarks {
720 open: Vec<bool>,
721 close: Vec<bool>,
722 folds: usize,
723}
724
725/// For every `Start` in `events`, the index of its matching `Finish`; `usize::MAX`
726/// for any unmatched opener. One stack pass, so a later walk can skip a whole
727/// subtree in constant time instead of rescanning it for the matching close.
728fn subtree_extents(events: &[Event]) -> Vec<usize> {
729 let mut extents = vec![usize::MAX; events.len()];
730 let mut open: Vec<usize> = Vec::new();
731 for (i, event) in events.iter().enumerate() {
732 match event {
733 Event::Start(_) => open.push(i),
734 Event::Finish => {
735 if let Some(start) = open.pop() {
736 extents[start] = i;
737 }
738 }
739 Event::Tok(_) => {}
740 }
741 }
742 extents
743}
744
745/// Walk the direct children of the level spanning `start..end`, recording a fold
746/// for each docstring pair and descending into every subtree. `is_container` says
747/// whether a fold may happen at *this* level.
748fn mark_doc_folds(
749 events: &[Event],
750 tokens: &[Token],
751 extents: &[usize],
752 start: usize,
753 end: usize,
754 is_container: bool,
755 marks: &mut DocMarks,
756) {
757 let mut i = start;
758 while i < end {
759 let Event::Start(kind) = events[i] else {
760 i += 1;
761 continue;
762 };
763 // An unmatched opener runs to the end of this level.
764 let stop = extents[i].min(end);
765 mark_doc_folds(
766 events,
767 tokens,
768 extents,
769 i + 1,
770 stop,
771 is_doc_container(kind),
772 marks,
773 );
774
775 if is_container
776 && kind == SyntaxKind::STRING_LITERAL
777 && string_is_doc_eligible(&events[i + 1..stop], tokens)
778 && let Some(target) = doc_target(events, tokens, stop + 1, end)
779 {
780 let target_end = extents[target].min(end);
781 // The target is consumed by this fold: it is documented *by* the
782 // string, so it never goes on to document what follows it. Descend
783 // into it here, then resume past it.
784 mark_doc_folds(
785 events,
786 tokens,
787 extents,
788 target + 1,
789 target_end,
790 is_doc_container(as_start_kind(&events[target])),
791 marks,
792 );
793 marks.open[i] = true;
794 marks.close[target_end] = true;
795 marks.folds += 1;
796 i = target_end + 1;
797 continue;
798 }
799 i = stop + 1;
800 }
801}
802
803/// The kind of a `Start` event. Only ever called on an event already matched as
804/// one, so the fallback kind is unreachable.
805fn as_start_kind(event: &Event) -> SyntaxKind {
806 match event {
807 Event::Start(kind) => *kind,
808 _ => SyntaxKind::ERROR,
809 }
810}
811
812/// Scanning forward from `from` within a level ending at `end`, the index of the
813/// `Start` of the statement a preceding docstring documents: the next subtree
814/// child, reachable across at most one newline of trivia and no `;`. `None` if no
815/// eligible target follows.
816fn doc_target(events: &[Event], tokens: &[Token], from: usize, end: usize) -> Option<usize> {
817 let mut newlines = 0;
818 let mut j = from;
819 while j < end {
820 match events[j] {
821 // An error-recovery node is never a documentable target (`"doc"\n]`
822 // ⇒ `(string) (error) (error-t ✘)`, not a `(doc …)`); the string is
823 // a plain statement.
824 Event::Start(SyntaxKind::ERROR) => return None,
825 Event::Start(_) => return Some(j),
826 Event::Tok(t) => {
827 let kind = tokens[t].kind;
828 if kind == TokKind::Newline {
829 newlines += 1;
830 if newlines > 1 {
831 return None;
832 }
833 } else if !kind.is_trivia() {
834 return None;
835 }
836 j += 1;
837 }
838 // Only reachable on an unbalanced stream, where it ends the level.
839 Event::Finish => return None,
840 }
841 }
842 None
843}
844
845/// Round-trip the input through the parser: concatenating every token in the CST
846/// must reproduce the original text exactly (the losslessness invariant).
847pub fn reconstruct(text: &str) -> String {
848 parse(text)
849 .cst
850 .descendants_with_tokens()
851 .filter_map(|el| el.into_token())
852 .map(|tok| tok.text().to_string())
853 .collect()
854}
855
856#[cfg(test)]
857mod tests {
858 use super::*;
859
860 fn assert_lossless(input: &str) {
861 assert_eq!(reconstruct(input), input, "not lossless: {input:?}");
862 }
863
864 #[test]
865 fn lossless_corpus() {
866 for input in [
867 "",
868 "x = 1\n",
869 "y = a + b * c\n",
870 "f(a, b, c)\n",
871 "v[1]\n",
872 "a[end]\n",
873 "a[end - 1]\n",
874 "a[2:end]\n",
875 "m[end, end]\n",
876 "z = (a + b) / 2\n",
877 "function g(x)\n x ^ 2\nend\n",
878 "if a >= b\n a\nelseif c\n c\nelse\n b\nend\n",
879 "begin\n a\n b\nend\n",
880 "map(xs) do x\n x + 1\nend\n",
881 "open(\"f\") do\n read()\nend\n",
882 "# a comment\nx = 1 # trailing\n",
883 "#= block =#\nq = 2\n",
884 "obj.field\n",
885 "x::Int\n",
886 "Vector{T}\n",
887 "foo(x::T) where {T<:Number} = x\n",
888 "f(a, b; c=1, d=2)\n",
889 "g(args...; kwargs...)\n",
890 "y = a .+ b .* c\n",
891 "z = .-x\n",
892 "a .= f.(b, c)\n",
893 "t = (a, b)\n",
894 "u = (a,)\n",
895 "e = ()\n",
896 "nt = (x = 1, y = 2)\n",
897 "y = a ? b : c\n",
898 "z = a == b ? x + 1 : x - 1\n",
899 "w = a ? b : c ? d : e\n",
900 "v = [1, 2, 3]\n",
901 "e = []\n",
902 "m = [1 2; 3 4]\n",
903 "s = [1 +2]\n",
904 "c = [1; 2; 3]\n",
905 "q = [x for x in xs]\n",
906 "r = [x^2 for x in 1:10 if x > 2]\n",
907 "g = (x for x in xs)\n",
908 "a; b\n",
909 "a;;;b;;\n",
910 ";a\n",
911 "@assert nl > 1 # trailing comment\nnx = 2\n",
912 "@foo x # c\n",
913 "x = @bind 🐶 html\"<input>\"\n",
914 "begin @m 10 end\n",
915 ] {
916 assert_lossless(input);
917 }
918 }
919
920 #[test]
921 fn groups_top_level_semicolon_line() {
922 let out = parse("a; b\nc\n");
923 let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
924 use crate::syntax::SyntaxKind::{NAME, TOPLEVEL_SEMICOLON};
925 assert_eq!(kinds, vec![TOPLEVEL_SEMICOLON, NAME]);
926 }
927
928 #[test]
929 fn builds_expected_top_level_shape() {
930 let out = parse("x = 1 + 2\n");
931 let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
932 assert_eq!(kinds, vec![crate::syntax::SyntaxKind::ASSIGNMENT_EXPR]);
933 assert!(out.diagnostics.is_empty());
934 }
935
936 #[test]
937 fn overflowing_decimal_floats_are_parse_errors() {
938 let out = parse("x=1e400\ny=1f400\nz=1e-4000\n");
939 assert_eq!(
940 out.diagnostics
941 .iter()
942 .map(|diagnostic| (
943 diagnostic.start,
944 diagnostic.end,
945 diagnostic.message.as_str()
946 ))
947 .collect::<Vec<_>>(),
948 vec![
949 (2, 7, "overflow in floating point literal"),
950 (10, 15, "overflow in floating point literal"),
951 ]
952 );
953 }
954}