fatou_parser/parser/core.rs
1use crate::parser::events::Event;
2use crate::parser::expr::parse_stmt;
3use crate::parser::lexer::{TokKind, Token, lex};
4use crate::parser::tree_builder::{build_tree, syntax_kind_for};
5use crate::syntax::{SyntaxKind, SyntaxNode};
6
7pub use crate::parser::diagnostics::ParseDiagnostic;
8use crate::parser::diagnostics::{DiagnosticKind, push_diagnostic};
9
10/// The result of a parse: the lossless CST plus any diagnostics.
11#[derive(Debug, Clone)]
12pub struct ParseOutput {
13 pub cst: SyntaxNode,
14 pub diagnostics: Vec<ParseDiagnostic>,
15}
16
17/// Parse `text` into a lossless `rowan` CST. The drive loop works one logical
18/// line at a time: leading trivia (including newlines) is emitted directly at
19/// the root, then the line's statements are parsed with the Pratt parser. A line
20/// that carries a top-level `;` groups its statements into a `TOPLEVEL_SEMICOLON`
21/// node (mirroring JuliaSyntax's `toplevel-;`); a plain line stays bare. An
22/// unparseable token is consumed as one error token so the loop always makes
23/// progress.
24pub fn parse(text: &str) -> ParseOutput {
25 let tokens = lex(text);
26 let mut diagnostics = Vec::new();
27 let mut events = Vec::new();
28
29 let mut i = 0usize;
30 while i < tokens.len() {
31 // Leading trivia and blank/newline runs belong directly at the root.
32 if tokens[i].kind.is_trivia() {
33 events.push(Event::Tok(i));
34 i += 1;
35 continue;
36 }
37
38 // Collect a single logical line (up to the next newline or EOF),
39 // tracking whether it carries a `;` separator. `leftover_mark` records
40 // the event offset right after the line's first complete statement so a
41 // trailing junk run on a separator-less line can be wrapped in an
42 // `ERROR` node and flagged with a `TrailingJunk` diagnostic (projected
43 // `(error-t …)`: `x y` ⇒ `x (error-t y)`).
44 let mut line = Vec::new();
45 let mut has_semicolon = false;
46 let mut leftover_mark: Option<usize> = None;
47 let mut first_is_doc_string = false;
48 // A doc-eligible string is only a docstring when a *real* statement
49 // follows it. If the leftover after the string can't start a statement
50 // (a stray closer/keyword: `"doc" ]`), the string is a plain statement
51 // and the remainder is junk — un-gate the raw trailing-junk collection.
52 let mut doc_no_target = false;
53 // A separator-less line: after the first complete statement, JuliaSyntax
54 // bumps the remainder as flat error tokens rather than re-parsing it. A
55 // line that carries a `;` keeps the per-segment behavior (deferred), so
56 // flat-junk collection is gated off when one is present.
57 let line_has_semicolon = rest_of_line_has_semicolon(&tokens, i);
58 while i < tokens.len() {
59 match tokens[i].kind {
60 TokKind::Newline => break,
61 TokKind::Semicolon => {
62 has_semicolon = true;
63 line.push(Event::Tok(i));
64 i += 1;
65 }
66 k if k.is_trivia() => {
67 line.push(Event::Tok(i));
68 i += 1;
69 }
70 _ => {
71 if !line_has_semicolon
72 && leftover_mark.is_some()
73 && (!first_is_doc_string || doc_no_target)
74 {
75 // Trailing junk after the first statement: collect raw
76 // (no structural re-parse) so the wrapping below puts the
77 // whole run in one ERROR node and the projector renders
78 // delimiters, commas, and `@` as `✘` (`x y, z` ⇒
79 // `x (error-t y ✘ z)`, `x@y` ⇒ `x (error-t ✘ y)`).
80 line.push(Event::Tok(i));
81 i += 1;
82 } else if let Some(expr) = parse_stmt(&tokens, i, &mut diagnostics) {
83 if leftover_mark.is_none() {
84 first_is_doc_string = stmt_is_doc_string(&expr.events, &tokens);
85 }
86 line.extend(expr.events);
87 i = expr.end;
88 leftover_mark.get_or_insert(line.len());
89 } else if leftover_mark.is_none()
90 && is_close_delimiter_tok(tokens[i].kind)
91 && !rest_of_line_has_semicolon(&tokens, i)
92 {
93 // A stray *closing* delimiter at statement start (no
94 // preceding statement) is JuliaSyntax's leading empty
95 // `(error)` plus an `(error-t ✘ …)` that swallows the
96 // rest of the line: `)` ⇒ `(error) (error-t ✘)`,
97 // `) x` ⇒ `(error) (error-t ✘ x)`, `)))` ⇒ `… ✘ ✘ ✘`.
98 // The `;`-segment forms emit a subtler double marker, so
99 // they are left to the loose-token fallback below.
100 line.push(Event::Start(SyntaxKind::ERROR));
101 line.push(Event::Finish);
102 let run_start = tokens[i].start;
103 line.push(Event::Start(SyntaxKind::ERROR));
104 while i < tokens.len() && tokens[i].kind != TokKind::Newline {
105 line.push(Event::Tok(i));
106 i += 1;
107 }
108 line.push(Event::Finish);
109 push_diagnostic(
110 &mut diagnostics,
111 DiagnosticKind::StrayCloser,
112 "stray closing delimiter",
113 run_start,
114 tokens[i - 1].end,
115 );
116 } else if leftover_mark.is_none() && is_stray_block_keyword_tok(tokens[i].kind)
117 {
118 // A middle/closing block keyword (`end`, `else`,
119 // `elseif`, `catch`, `finally`) where a statement is
120 // expected is not a block opener; JuliaSyntax wraps it
121 // alone in `(error <kw>)` and bumps the rest of the line
122 // as a separate trailing-junk run (`end y z` ⇒
123 // `(error end) (error-t y z)`). Emit the wrapped keyword
124 // as the line's first "statement" and set `leftover_mark`
125 // so the existing trailing-junk machinery handles the
126 // remainder.
127 line.push(Event::Start(SyntaxKind::ERROR));
128 line.push(Event::Tok(i));
129 line.push(Event::Finish);
130 push_diagnostic(
131 &mut diagnostics,
132 DiagnosticKind::StrayKeyword,
133 "unexpected block keyword",
134 tokens[i].start,
135 tokens[i].end,
136 );
137 i += 1;
138 leftover_mark = Some(line.len());
139 } else {
140 // A leftover token that can't start a statement. After a
141 // doc-eligible string this means the string has no
142 // documentable target, so the rest of the line is junk.
143 if first_is_doc_string && leftover_mark.is_some() {
144 doc_no_target = true;
145 }
146 line.push(Event::Tok(i));
147 i += 1;
148 }
149 }
150 }
151 }
152
153 if has_semicolon {
154 events.push(Event::Start(SyntaxKind::TOPLEVEL_SEMICOLON));
155 events.extend(line);
156 events.push(Event::Finish);
157 } else if let Some(mark) = leftover_mark.filter(|&m| {
158 // Junk after the first statement on a separator-less line — but a
159 // bare docstring (`"a"\nfoo`) owns its trailing statement, so leave
160 // that to `fold_docstrings`. A docstring defers only when its
161 // leftover actually begins with a documentable statement subtree
162 // (`"doc" foo`); a leftover that opens with junk (`"doc" ]`) is
163 // recovered here so the stray closer becomes `(error-t ✘)`.
164 let tail = &line[m..];
165 let defer_to_doc = first_is_doc_string && leftover_starts_with_subtree(tail, &tokens);
166 !defer_to_doc && tail.iter().any(|e| is_significant_event(e, &tokens))
167 }) {
168 events.extend(line[..mark].iter().cloned());
169 let tail = &line[mark..];
170 // Leading trivia stays outside the error node; everything from the
171 // first significant leftover token onward is the recovered run.
172 let lead = tail
173 .iter()
174 .take_while(|e| !is_significant_event(e, &tokens))
175 .count();
176 events.extend(tail[..lead].iter().cloned());
177 let first_junk = tail[lead..].iter().find_map(|e| match e {
178 Event::Tok(idx) => Some(*idx),
179 _ => None,
180 });
181 events.push(Event::Start(SyntaxKind::ERROR));
182 events.extend(tail[lead..].iter().cloned());
183 events.push(Event::Finish);
184 if let Some(idx) = first_junk {
185 push_diagnostic(
186 &mut diagnostics,
187 DiagnosticKind::TrailingJunk,
188 "trailing tokens after statement",
189 tokens[idx].start,
190 tokens[idx].end,
191 );
192 }
193 } else {
194 events.extend(line);
195 }
196 }
197
198 let events = fold_docstrings(&events, &tokens, true);
199
200 let cst = build_tree(&tokens, &events);
201 flag_invalid_const_decls(&cst, &mut diagnostics);
202 flag_invalid_function_signatures(&cst, &mut diagnostics);
203 flag_invalid_catch_vars(&cst, &mut diagnostics);
204 flag_invalid_export_items(&cst, &mut diagnostics);
205 ParseOutput { cst, diagnostics }
206}
207
208/// Flag each parenthesized `export` item that is not a single symbol. An export
209/// item may be parenthesized, but only around a single identifier, operator,
210/// `var"…"`, or `$`-interpolation (`export (x)`/`export (+)`/`export ($a)`),
211/// which JuliaSyntax unwraps to the bare symbol. Any other parenthesized form
212/// (`export (x::T)`, `export (x, y)`, `export ()`, `export ((x))`) is invalid and
213/// JuliaSyntax error-wraps the parsed expression (`(export (error (::-i x T)))`).
214/// The diagnostic is a zero-width point at the parenthesized node's start; the
215/// projector reconstructs the error wrapper from it (the CST stays faithful).
216fn flag_invalid_export_items(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
217 for stmt in cst
218 .descendants()
219 .filter(|n| n.kind() == SyntaxKind::EXPORT_STMT)
220 {
221 for item in stmt
222 .children()
223 .filter(|c| matches!(c.kind(), SyntaxKind::PAREN_EXPR | SyntaxKind::TUPLE_EXPR))
224 {
225 if !export_paren_is_symbol(&item) {
226 let pos = usize::from(item.text_range().start());
227 push_diagnostic(
228 diagnostics,
229 DiagnosticKind::InvalidExportItem,
230 "invalid `export` item: expected a single symbol",
231 pos,
232 pos,
233 );
234 }
235 }
236 }
237}
238
239/// Whether a parenthesized `export` item wraps a single symbol that JuliaSyntax
240/// unwraps: a lone identifier (`(x)`), `var"…"` (`(var"x")`), `$`-interpolation
241/// (`($a)`), or operator (`(+)`). A `(x, y)` tuple, an empty `()`, a nested paren
242/// (`((x))`), a literal (`(1)`), or any compound expression (`(x::T)`, `(a+b)`)
243/// is not. A `TUPLE_EXPR` (a comma/empty paren) is always invalid.
244fn export_paren_is_symbol(item: &SyntaxNode) -> bool {
245 if item.kind() != SyntaxKind::PAREN_EXPR {
246 return false;
247 }
248 let mut significant = item.children_with_tokens().filter(|el| {
249 !matches!(
250 el.kind(),
251 SyntaxKind::WHITESPACE
252 | SyntaxKind::NEWLINE
253 | SyntaxKind::COMMENT
254 | SyntaxKind::BLOCK_COMMENT
255 | SyntaxKind::LPAREN
256 | SyntaxKind::RPAREN
257 )
258 });
259 let Some(first) = significant.next() else {
260 return false;
261 };
262 if significant.next().is_some() {
263 return false;
264 }
265 match first {
266 // A symbol-shaped node: a bare name, `var"…"`, or `$`-interpolation. A
267 // `LITERAL`, `TYPE_ANNOTATION`, nested `PAREN_EXPR`, etc. is not.
268 rowan::NodeOrToken::Node(n) => match n.kind() {
269 SyntaxKind::NAME | SyntaxKind::NONSTANDARD_IDENTIFIER | SyntaxKind::INTERPOLATION => {
270 true
271 }
272 // A parenthesized operator value (`(..)`, `(√)`, `(⊕)`) — which
273 // `parse_paren` wraps in an `OPERATOR_ATOM` — is a valid export
274 // symbol, but a dotted broadcast operator (`(.+)`, `(.≤)`) is not:
275 // JuliaSyntax error-wraps it (`export (.+)` ⇒ `(export (error (. +)))`).
276 SyntaxKind::OPERATOR_ATOM => !export_operator_atom_is_dotted(&n),
277 _ => false,
278 },
279 // The only bare token a `PAREN_EXPR` carries is the lone operator-value
280 // form (`(+)`, `(:)`), which `parse_paren` builds for `is_paren_value_op`.
281 rowan::NodeOrToken::Token(_) => true,
282 }
283}
284
285/// Whether a parenthesized `export`/`import` operator atom is a *dotted*
286/// (broadcast) operator (`.+`, `.≤`, `.&&`), which JuliaSyntax rejects as an
287/// export symbol. The range/splat operators `..`/`...` lead with a *doubled*
288/// dot and are not broadcasts, so they are valid; a lone `√`/`⊕` has no dot.
289fn export_operator_atom_is_dotted(atom: &SyntaxNode) -> bool {
290 atom.children_with_tokens()
291 .filter_map(|el| el.into_token())
292 .find(|t| {
293 !matches!(
294 t.kind(),
295 SyntaxKind::WHITESPACE
296 | SyntaxKind::NEWLINE
297 | SyntaxKind::COMMENT
298 | SyntaxKind::BLOCK_COMMENT
299 )
300 })
301 .is_some_and(|t| {
302 let bytes = t.text().as_bytes();
303 bytes.first() == Some(&b'.') && bytes.len() > 1 && bytes[1] != b'.'
304 })
305}
306
307/// Flag each `const` whose declaration is not the plain `=` assignment
308/// JuliaSyntax requires. A bare declaration (`const x`), a non-`=` assignment
309/// (`const x += 1`, `const x .= 1`), or a `global`/`local` declaration without an
310/// `=` (`const global x`) is invalid, so JuliaSyntax wraps the whole `const` in
311/// `(error …)` (`const x` ⇒ `(error (const x))`). A valid `const x = 1` — or a
312/// `global`/`local`-wrapped `=` (`const global x = 1`) — is left alone. The
313/// diagnostic is a zero-width point at the `const` keyword start; the projector
314/// reconstructs the error wrapper from it (the CST topology stays faithful).
315fn flag_invalid_const_decls(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
316 for node in cst
317 .descendants()
318 .filter(|n| n.kind() == SyntaxKind::CONST_STMT)
319 {
320 if !const_decl_is_assignment(&node) && !is_struct_const_field(&node) {
321 let pos = usize::from(node.text_range().start());
322 push_diagnostic(
323 diagnostics,
324 DiagnosticKind::ConstNotAssignment,
325 "expected assignment after `const`",
326 pos,
327 pos,
328 );
329 }
330 }
331}
332
333/// Flag each `function`/`macro` whose signature is a bare identifier name but
334/// which carries a non-empty body. A header like `function f end` (a bare name
335/// with a truly empty body) is the valid forward-declaration form `(function f)`,
336/// but once a body is present (`function f body end`) or the body block is
337/// explicitly opened with a `;` (`function f; end`), the bare name is no longer a
338/// valid signature: JuliaSyntax error-wraps it (`(function (error f) (block
339/// body))`). The diagnostic is a zero-width point at the `SIGNATURE` node's start;
340/// the projector reconstructs the error wrapper from it (the CST stays faithful).
341fn flag_invalid_function_signatures(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
342 for node in cst
343 .descendants()
344 .filter(|n| matches!(n.kind(), SyntaxKind::FUNCTION_DEF | SyntaxKind::MACRO_DEF))
345 {
346 let Some(sig) = node.children().find(|c| c.kind() == SyntaxKind::SIGNATURE) else {
347 continue;
348 };
349 let sig_is_bare_name = sig
350 .first_child()
351 .is_some_and(|inner| is_bare_signature_name(&inner));
352 if sig_is_bare_name && !function_body_is_empty(&node) {
353 let pos = usize::from(sig.text_range().start());
354 push_diagnostic(
355 diagnostics,
356 DiagnosticKind::InvalidFunctionSignature,
357 "invalid function signature: expected a call, not a bare name",
358 pos,
359 pos,
360 );
361 }
362 }
363}
364
365/// Whether a `function`/`macro` signature's first node is a *bare name* — the
366/// thing being declared rather than a call or other expression. Besides a plain
367/// identifier and a `$`-interpolation, an operator name counts (`function + end`,
368/// `function .+ end`), including one Julia rejects as a name, which the parser
369/// leaves as an `ERROR`-wrapped operator atom (`function = end` ⇒
370/// `(function (error =))`).
371pub(super) fn is_bare_signature_name(node: &SyntaxNode) -> bool {
372 match node.kind() {
373 SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::OPERATOR_ATOM => true,
374 SyntaxKind::ERROR => node
375 .first_child()
376 .is_some_and(|inner| inner.kind() == SyntaxKind::OPERATOR_ATOM),
377 _ => false,
378 }
379}
380
381/// Flag each `catch` clause whose variable is not a plain identifier. The catch
382/// variable must be a bare identifier (`catch e`), a `$`-interpolation
383/// (`catch $e`), or a `var"…"` non-standard identifier (`catch var"e"`); any
384/// other expression (`catch e+3`, `catch e.f`, `catch f(e)`, `catch 3`) is
385/// invalid and JuliaSyntax wraps it in `(error …)` (`catch e+3` ⇒ `(catch
386/// (error (call-i e + 3)) …)`). The diagnostic is a zero-width point at the
387/// catch-variable node's start; the projector reconstructs the error wrapper
388/// from it (the CST topology stays faithful).
389fn flag_invalid_catch_vars(cst: &SyntaxNode, diagnostics: &mut Vec<ParseDiagnostic>) {
390 for node in cst
391 .descendants()
392 .filter(|n| n.kind() == SyntaxKind::CATCH_CLAUSE)
393 {
394 let Some(var) = node.children().find(|c| c.kind() != SyntaxKind::BLOCK) else {
395 continue;
396 };
397 if !matches!(
398 var.kind(),
399 SyntaxKind::NAME | SyntaxKind::INTERPOLATION | SyntaxKind::NONSTANDARD_IDENTIFIER
400 ) {
401 let pos = usize::from(var.text_range().start());
402 push_diagnostic(
403 diagnostics,
404 DiagnosticKind::CatchVarNotIdentifier,
405 "catch variable must be an identifier",
406 pos,
407 pos,
408 );
409 }
410 }
411}
412
413/// Whether a `function`/`macro`'s body is syntactically empty — no statement
414/// nodes and no `;` separator opening the block. A bare-name header keeps the
415/// forward-declaration form only while this holds; a `;` (`function f; end`) or
416/// any body statement marks the block as genuinely opened.
417fn function_body_is_empty(node: &SyntaxNode) -> bool {
418 match node.children().find(|c| c.kind() == SyntaxKind::BLOCK) {
419 Some(block) => {
420 block.first_child().is_none()
421 && !block
422 .children_with_tokens()
423 .any(|el| el.kind() == SyntaxKind::SEMICOLON)
424 }
425 None => true,
426 }
427}
428
429/// Whether `node` is a `const` field declaration directly inside a struct body
430/// (`struct A const a end` ⇒ `(const a)`, not wrapped). Such a bare `const` is
431/// valid; the exemption is narrow — a `const` nested inside an `if`/`begin`
432/// within the struct (its block's parent is not the `STRUCT_DEF`) is still an
433/// error.
434fn is_struct_const_field(node: &SyntaxNode) -> bool {
435 node.parent()
436 .filter(|b| b.kind() == SyntaxKind::BLOCK)
437 .and_then(|b| b.parent())
438 .is_some_and(|p| p.kind() == SyntaxKind::STRUCT_DEF)
439}
440
441/// Whether a `const`'s declaration is a plain `=` assignment. The body is the
442/// `const`'s first child node, unwrapping any `global`/`local` declaration; it is
443/// valid only when it is an `ASSIGNMENT_EXPR` headed by a plain `=` (not
444/// `.=`/`+=`/…).
445fn const_decl_is_assignment(node: &SyntaxNode) -> bool {
446 let mut body = node.first_child();
447 while let Some(n) = body {
448 match n.kind() {
449 SyntaxKind::GLOBAL_STMT | SyntaxKind::LOCAL_STMT => body = n.first_child(),
450 SyntaxKind::ASSIGNMENT_EXPR => {
451 return n
452 .children_with_tokens()
453 .filter_map(|el| el.into_token())
454 .any(|t| t.kind() == SyntaxKind::EQ);
455 }
456 _ => return false,
457 }
458 }
459 false
460}
461
462/// Whether `kind` is a closing delimiter (`)`, `]`, `}`), which when stray at
463/// statement start drives JuliaSyntax's leading-`(error)` recovery.
464fn is_close_delimiter_tok(kind: TokKind) -> bool {
465 matches!(kind, TokKind::RParen | TokKind::RBracket | TokKind::RBrace)
466}
467
468/// Whether `kind` is a middle/closing block keyword (`end`, `else`, `elseif`,
469/// `catch`, `finally`) — a token that only closes or continues an enclosing
470/// block. Where a statement is expected one of these is stray, so JuliaSyntax
471/// wraps it alone in `(error <kw>)` (`@doc x\nend` ⇒ `(macrocall @doc x) (error
472/// end)`, `end y z` ⇒ `(error end) (error-t y z)`).
473fn is_stray_block_keyword_tok(kind: TokKind) -> bool {
474 matches!(
475 kind,
476 TokKind::EndKw
477 | TokKind::ElseKw
478 | TokKind::ElseifKw
479 | TokKind::CatchKw
480 | TokKind::FinallyKw
481 )
482}
483
484/// Whether the logical line starting at `start` (up to the next newline or EOF)
485/// carries a top-level `;`. The stray-closer recovery only applies to the clean
486/// separator-less form; `;`-segment lines keep the loose-token fallback.
487fn rest_of_line_has_semicolon(tokens: &[Token], start: usize) -> bool {
488 tokens[start..]
489 .iter()
490 .take_while(|t| t.kind != TokKind::Newline)
491 .any(|t| t.kind == TokKind::Semicolon)
492}
493
494/// Whether an event carries significant (non-trivia) content: any node opener,
495/// or a non-trivia leaf token.
496fn is_significant_event(event: &Event, tokens: &[Token]) -> bool {
497 match event {
498 Event::Start(_) => true,
499 Event::Tok(idx) => !tokens[*idx].kind.is_trivia(),
500 Event::Finish => false,
501 }
502}
503
504/// Whether the first significant event in `tail` opens a node (a real statement
505/// subtree). A docstring's trailing leftover defers to `fold_docstrings` only
506/// when it begins with such a subtree (its documentable target); a leftover that
507/// opens with a bare junk token is recovered as trailing junk instead.
508fn leftover_starts_with_subtree(tail: &[Event], tokens: &[Token]) -> bool {
509 matches!(
510 tail.iter().find(|e| is_significant_event(e, tokens)),
511 Some(Event::Start(_))
512 )
513}
514
515/// Whether a statement's events open a bare (doc-eligible) `STRING_LITERAL` —
516/// the first inner token is not a `STRING_PREFIX`. Such a statement starts a
517/// potential docstring, so a trailing statement on the same logical line is left
518/// to `fold_docstrings` rather than wrapped as junk.
519pub(super) fn stmt_is_doc_string(events: &[Event], tokens: &[Token]) -> bool {
520 matches!(
521 events.first(),
522 Some(Event::Start(SyntaxKind::STRING_LITERAL))
523 ) && string_is_doc_eligible(&events[1..], tokens)
524}
525
526/// One child of a statement container: either a leaf token or a fully-formed
527/// subtree (its node kind plus the *inner* events between its `Start`/`Finish`).
528enum Item {
529 Leaf(usize),
530 Subtree(SyntaxKind, Vec<Event>),
531}
532
533/// Whether a node of `kind` is a statement container in which the docstring fold
534/// applies (a string literal statement directly followed by another statement).
535fn is_doc_container(kind: SyntaxKind) -> bool {
536 matches!(kind, SyntaxKind::BLOCK | SyntaxKind::TOPLEVEL_SEMICOLON)
537}
538
539/// Whether a `STRING_LITERAL` subtree is a plain string (doc-eligible) rather
540/// than a prefixed string macro (`r"…"`, `b"…"`), which is a macro call in
541/// JuliaSyntax and never a docstring. Eligible iff its first token is not a
542/// `STRING_PREFIX`.
543fn string_is_doc_eligible(inner: &[Event], tokens: &[Token]) -> bool {
544 match inner.first() {
545 Some(Event::Tok(idx)) => syntax_kind_for(tokens[*idx].kind) != SyntaxKind::STRING_PREFIX,
546 _ => true,
547 }
548}
549
550/// Recursively fold docstrings in a statement container's child sequence.
551/// `inner` is the flat event list of one node's children (the whole event
552/// stream, for the implicit `ROOT`). A bare unprefixed `STRING_LITERAL`
553/// statement immediately followed by another statement — at most one newline of
554/// intervening trivia, no `;` — folds into a `DOC` node `(doc str target)`,
555/// mirroring JuliaSyntax's `parse_docstring`. The pass descends into every
556/// subtree so nested blocks (function/module/begin bodies) fold too.
557fn fold_docstrings(inner: &[Event], tokens: &[Token], is_container: bool) -> Vec<Event> {
558 // Split the flat event list into this level's direct children.
559 let mut items: Vec<Item> = Vec::new();
560 let mut k = 0;
561 while k < inner.len() {
562 match inner[k] {
563 Event::Tok(idx) => {
564 items.push(Item::Leaf(idx));
565 k += 1;
566 }
567 Event::Start(kind) => {
568 let mut depth = 1usize;
569 let mut j = k + 1;
570 while j < inner.len() {
571 match inner[j] {
572 Event::Start(_) => depth += 1,
573 Event::Finish => {
574 depth -= 1;
575 if depth == 0 {
576 break;
577 }
578 }
579 Event::Tok(_) => {}
580 }
581 j += 1;
582 }
583 // Recurse into the subtree's inner events.
584 let child = fold_docstrings(&inner[k + 1..j], tokens, is_doc_container(kind));
585 items.push(Item::Subtree(kind, child));
586 k = j + 1;
587 }
588 Event::Finish => k += 1,
589 }
590 }
591
592 let mut out: Vec<Event> = Vec::new();
593 let mut idx = 0;
594 while idx < items.len() {
595 if is_container
596 && let Item::Subtree(SyntaxKind::STRING_LITERAL, str_inner) = &items[idx]
597 && string_is_doc_eligible(str_inner, tokens)
598 && let Some(target) = doc_target(&items, idx, tokens)
599 {
600 // Wrap the string, the intervening trivia, and the target in a `DOC`.
601 out.push(Event::Start(SyntaxKind::DOC));
602 emit_items(&items[idx..=target], &mut out);
603 out.push(Event::Finish);
604 idx = target + 1;
605 continue;
606 }
607 emit_items(&items[idx..=idx], &mut out);
608 idx += 1;
609 }
610 out
611}
612
613/// Given a string statement at `start`, find the index of the statement it
614/// documents: the next subtree child, reachable across at most one newline of
615/// trivia and no `;`. Returns `None` if no eligible target follows.
616fn doc_target(items: &[Item], start: usize, tokens: &[Token]) -> Option<usize> {
617 let mut newlines = 0;
618 let mut j = start + 1;
619 while j < items.len() {
620 match &items[j] {
621 // An error-recovery node is never a documentable target (`"doc"\n]`
622 // ⇒ `(string) (error) (error-t ✘)`, not a `(doc …)`); the string is
623 // a plain statement.
624 Item::Subtree(SyntaxKind::ERROR, _) => return None,
625 Item::Subtree(..) => return Some(j),
626 Item::Leaf(t) => {
627 let kind = tokens[*t].kind;
628 if kind == TokKind::Newline {
629 newlines += 1;
630 if newlines > 1 {
631 return None;
632 }
633 } else if !kind.is_trivia() {
634 return None;
635 }
636 j += 1;
637 }
638 }
639 }
640 None
641}
642
643/// Append the events for `items` to `out`, rebuilding each subtree from its
644/// recorded kind and inner events.
645fn emit_items(items: &[Item], out: &mut Vec<Event>) {
646 for item in items {
647 match item {
648 Item::Leaf(idx) => out.push(Event::Tok(*idx)),
649 Item::Subtree(kind, inner) => {
650 out.push(Event::Start(*kind));
651 out.extend(inner.iter().cloned());
652 out.push(Event::Finish);
653 }
654 }
655 }
656}
657
658/// Round-trip the input through the parser: concatenating every token in the CST
659/// must reproduce the original text exactly (the losslessness invariant).
660pub fn reconstruct(text: &str) -> String {
661 parse(text)
662 .cst
663 .descendants_with_tokens()
664 .filter_map(|el| el.into_token())
665 .map(|tok| tok.text().to_string())
666 .collect()
667}
668
669#[cfg(test)]
670mod tests {
671 use super::*;
672
673 fn assert_lossless(input: &str) {
674 assert_eq!(reconstruct(input), input, "not lossless: {input:?}");
675 }
676
677 #[test]
678 fn lossless_corpus() {
679 for input in [
680 "",
681 "x = 1\n",
682 "y = a + b * c\n",
683 "f(a, b, c)\n",
684 "v[1]\n",
685 "a[end]\n",
686 "a[end - 1]\n",
687 "a[2:end]\n",
688 "m[end, end]\n",
689 "z = (a + b) / 2\n",
690 "function g(x)\n x ^ 2\nend\n",
691 "if a >= b\n a\nelseif c\n c\nelse\n b\nend\n",
692 "begin\n a\n b\nend\n",
693 "map(xs) do x\n x + 1\nend\n",
694 "open(\"f\") do\n read()\nend\n",
695 "# a comment\nx = 1 # trailing\n",
696 "#= block =#\nq = 2\n",
697 "obj.field\n",
698 "x::Int\n",
699 "Vector{T}\n",
700 "foo(x::T) where {T<:Number} = x\n",
701 "f(a, b; c=1, d=2)\n",
702 "g(args...; kwargs...)\n",
703 "y = a .+ b .* c\n",
704 "z = .-x\n",
705 "a .= f.(b, c)\n",
706 "t = (a, b)\n",
707 "u = (a,)\n",
708 "e = ()\n",
709 "nt = (x = 1, y = 2)\n",
710 "y = a ? b : c\n",
711 "z = a == b ? x + 1 : x - 1\n",
712 "w = a ? b : c ? d : e\n",
713 "v = [1, 2, 3]\n",
714 "e = []\n",
715 "m = [1 2; 3 4]\n",
716 "s = [1 +2]\n",
717 "c = [1; 2; 3]\n",
718 "q = [x for x in xs]\n",
719 "r = [x^2 for x in 1:10 if x > 2]\n",
720 "g = (x for x in xs)\n",
721 "a; b\n",
722 "a;;;b;;\n",
723 ";a\n",
724 "@assert nl > 1 # trailing comment\nnx = 2\n",
725 "@foo x # c\n",
726 "x = @bind 🐶 html\"<input>\"\n",
727 "begin @m 10 end\n",
728 ] {
729 assert_lossless(input);
730 }
731 }
732
733 #[test]
734 fn groups_top_level_semicolon_line() {
735 let out = parse("a; b\nc\n");
736 let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
737 use crate::syntax::SyntaxKind::{NAME, TOPLEVEL_SEMICOLON};
738 assert_eq!(kinds, vec![TOPLEVEL_SEMICOLON, NAME]);
739 }
740
741 #[test]
742 fn builds_expected_top_level_shape() {
743 let out = parse("x = 1 + 2\n");
744 let kinds: Vec<_> = out.cst.children().map(|n| n.kind()).collect();
745 assert_eq!(kinds, vec![crate::syntax::SyntaxKind::ASSIGNMENT_EXPR]);
746 assert!(out.diagnostics.is_empty());
747 }
748}