1use std::cell::Cell;
21use std::sync::Arc;
22
23use cstree::Syntax;
24use cstree::build::GreenNodeBuilder;
25use cstree::green::GreenNode;
26use cstree::interning::TokenInterner;
27use cstree::syntax::ResolvedNode;
28use text_size::{TextRange, TextSize};
29
30use crate::SyntaxKind;
31use crate::lexer::{RawToken, tokenize};
32use crate::prepass::run as run_prepass;
33
34mod grammar;
35
36#[derive(Debug, Clone)]
39pub struct Parse {
40 green: GreenNode,
41 interner: Arc<TokenInterner>,
42 errors: Vec<SyntaxError>,
43}
44
45impl Parse {
46 #[must_use]
49 pub fn syntax_node(&self) -> ResolvedNode<SyntaxKind> {
50 ResolvedNode::new_root_with_resolver(self.green.clone(), Arc::clone(&self.interner))
51 }
52
53 #[must_use]
55 pub fn errors(&self) -> &[SyntaxError] {
56 &self.errors
57 }
58
59 #[must_use]
61 pub fn green(&self) -> &GreenNode {
62 &self.green
63 }
64
65 #[must_use]
68 pub fn debug_tree(&self) -> String {
69 cstree::syntax::SyntaxNode::<SyntaxKind>::new_root(self.green.clone())
70 .debug(&self.interner, true)
71 }
72}
73
74impl PartialEq for Parse {
81 fn eq(&self, other: &Self) -> bool {
82 self.green == other.green && self.errors == other.errors
83 }
84}
85impl Eq for Parse {}
86
87#[derive(Debug, Clone, PartialEq, Eq)]
89pub struct SyntaxError {
90 pub range: TextRange,
92 pub message: String,
94}
95
96#[must_use]
98pub fn parse(text: &str) -> Parse {
99 let raw = tokenize(text);
100 let (tokens, indent_diags) = run_prepass(&raw, text);
101
102 let mut p = Parser::new(text, &tokens);
103 p.source_file();
104 let Parser {
105 events, mut errors, ..
106 } = p;
107
108 errors.extend(indent_diags.into_iter().map(|d| SyntaxError {
109 range: d.range,
110 message: d.message,
111 }));
112
113 let (green, interner) = build_tree(&events, &tokens, text);
114 Parse {
115 green,
116 interner,
117 errors,
118 }
119}
120
121#[derive(Debug, Clone, Copy)]
124enum Event {
125 Open { kind: SyntaxKind },
126 Close,
127 Advance,
128}
129
130struct Marker {
132 pos: usize,
133}
134
135#[derive(Clone, Copy)]
138struct MarkClosed {
139 pos: usize,
140}
141
142const FUEL: u32 = 256;
146
147struct Parser<'s> {
148 src: &'s str,
149 tokens: &'s [RawToken],
150 nontrivia: Vec<usize>,
152 pos: usize,
154 fuel: Cell<u32>,
155 events: Vec<Event>,
156 errors: Vec<SyntaxError>,
157}
158
159impl<'s> Parser<'s> {
160 fn new(src: &'s str, tokens: &'s [RawToken]) -> Self {
161 let nontrivia = tokens
162 .iter()
163 .enumerate()
164 .filter(|(_, t)| !t.kind.is_trivia())
165 .map(|(i, _)| i)
166 .collect();
167 Self {
168 src,
169 tokens,
170 nontrivia,
171 pos: 0,
172 fuel: Cell::new(FUEL),
173 events: Vec::new(),
174 errors: Vec::new(),
175 }
176 }
177
178 fn nth(&self, n: usize) -> SyntaxKind {
180 assert!(self.fuel.get() > 0, "parser stuck at position {}", self.pos);
181 self.fuel.set(self.fuel.get() - 1);
182 self.nontrivia
183 .get(self.pos + n)
184 .map_or(SyntaxKind::Eof, |&i| self.tokens[i].kind)
185 }
186
187 fn at(&self, kind: SyntaxKind) -> bool {
188 self.nth(0) == kind
189 }
190
191 fn match_begins_statement(&self) -> bool {
200 use SyntaxKind as K;
201 let kind_at = |off: usize| {
202 self.nontrivia
203 .get(self.pos + off)
204 .map_or(K::Eof, |&i| self.tokens[i].kind)
205 };
206 match kind_at(1) {
207 K::Dot
209 | K::Eq
210 | K::ColonEq
211 | K::PlusEq
212 | K::MinusEq
213 | K::StarEq
214 | K::SlashEq
215 | K::StarStarEq
216 | K::PercentEq
217 | K::AmpEq
218 | K::PipeEq
219 | K::CaretEq
220 | K::ShlEq
221 | K::ShrEq => false,
222 K::LParen | K::LBrack => {
223 let mut depth = 0usize;
225 let mut off = 1;
226 loop {
227 match kind_at(off) {
228 K::Eof => return true, K::LParen | K::LBrack | K::LBrace => depth += 1,
230 K::RParen | K::RBrack | K::RBrace => {
231 depth -= 1;
232 if depth == 0 {
233 return kind_at(off + 1) == K::Colon;
236 }
237 }
238 _ => {}
239 }
240 off += 1;
241 }
242 }
243 _ => true,
244 }
245 }
246
247 fn at_any(&self, kinds: &[SyntaxKind]) -> bool {
248 kinds.contains(&self.nth(0))
249 }
250
251 fn eof(&self) -> bool {
252 self.pos >= self.nontrivia.len()
253 }
254
255 fn cur_range(&self) -> TextRange {
257 self.nontrivia.get(self.pos).map_or_else(
258 || TextRange::empty(TextSize::of(self.src)),
259 |&i| self.tokens[i].range,
260 )
261 }
262
263 fn cur_text(&self) -> &str {
266 self.nontrivia
267 .get(self.pos)
268 .map_or("", |&i| &self.src[self.tokens[i].range])
269 }
270
271 fn advance(&mut self) {
272 if self.eof() {
276 return;
277 }
278 self.fuel.set(FUEL);
279 self.events.push(Event::Advance);
280 self.pos += 1;
281 }
282
283 fn open(&mut self) -> Marker {
284 let m = Marker {
285 pos: self.events.len(),
286 };
287 self.events.push(Event::Open {
288 kind: SyntaxKind::Tombstone,
289 });
290 m
291 }
292
293 #[allow(clippy::needless_pass_by_value)]
296 fn close(&mut self, m: Marker, kind: SyntaxKind) -> MarkClosed {
297 self.events[m.pos] = Event::Open { kind };
298 self.events.push(Event::Close);
299 MarkClosed { pos: m.pos }
300 }
301
302 fn open_before(&mut self, m: MarkClosed) -> Marker {
305 self.events.insert(
306 m.pos,
307 Event::Open {
308 kind: SyntaxKind::Tombstone,
309 },
310 );
311 Marker { pos: m.pos }
312 }
313
314 fn eat(&mut self, kind: SyntaxKind) -> bool {
315 if self.at(kind) {
316 self.advance();
317 true
318 } else {
319 false
320 }
321 }
322
323 fn expect(&mut self, kind: SyntaxKind) {
325 if self.eat(kind) {
326 return;
327 }
328 self.error(format!("expected {kind:?}"));
329 }
330
331 fn error(&mut self, message: String) {
333 self.errors.push(SyntaxError {
334 range: self.cur_range(),
335 message,
336 });
337 }
338
339 fn advance_with_error(&mut self, message: &str) -> MarkClosed {
343 let m = self.open();
344 self.error(message.to_owned());
345 if !self.eof() {
346 self.advance();
347 }
348 self.close(m, SyntaxKind::ErrorNode)
349 }
350}
351
352fn build_tree(events: &[Event], tokens: &[RawToken], src: &str) -> (GreenNode, Arc<TokenInterner>) {
356 let mut builder: GreenNodeBuilder<'static, 'static, SyntaxKind> = GreenNodeBuilder::new();
357 let mut tok = 0usize;
358 let mut depth: u32 = 0;
359
360 for event in events {
361 match *event {
362 Event::Open { kind } => {
363 if kind == SyntaxKind::Tombstone {
364 continue; }
366 depth += 1;
367 builder.start_node(kind);
368 }
369 Event::Close => {
370 depth -= 1;
371 if depth == 0 {
372 while tok < tokens.len() {
375 emit(&mut builder, tokens[tok], src);
376 tok += 1;
377 }
378 }
379 builder.finish_node();
380 }
381 Event::Advance => {
382 while tok < tokens.len() && tokens[tok].kind.is_trivia() {
383 emit(&mut builder, tokens[tok], src);
384 tok += 1;
385 }
386 if tok < tokens.len() {
387 emit(&mut builder, tokens[tok], src);
388 tok += 1;
389 }
390 }
391 }
392 }
393
394 let (green, cache) = builder.finish();
395 let interner = cache
396 .expect("a builder created with `new()` owns its cache")
397 .into_interner()
398 .expect("the cache owns its interner");
399 (green, Arc::new(interner))
400}
401
402fn emit(builder: &mut GreenNodeBuilder<'static, 'static, SyntaxKind>, t: RawToken, src: &str) {
406 if <SyntaxKind as Syntax>::static_text(t.kind).is_some() {
407 builder.static_token(t.kind);
408 } else {
409 builder.token(t.kind, &src[t.range]);
410 }
411}
412
413#[cfg(test)]
414mod tests {
415 use super::*;
416
417 fn round_trips(src: &str) {
418 let parse = parse(src);
419 assert_eq!(
420 parse.syntax_node().to_string(),
421 src,
422 "round-trip mismatch for {src:?}",
423 );
424 }
425
426 #[test]
427 fn round_trips_a_function() {
428 round_trips("func f():\n\tpass\n");
429 }
430
431 #[test]
432 fn round_trips_inline_function() {
433 round_trips("func square(a): return a\n");
434 }
435
436 #[test]
437 fn round_trips_with_trivia() {
438 round_trips("## doc\nfunc _ready() -> void:\n\tpass\n\n# trailing comment\n");
439 }
440
441 #[test]
442 fn round_trips_multiple_functions() {
443 round_trips("func a():\n\tpass\nfunc b():\n\tpass\n");
444 }
445
446 #[test]
447 fn round_trips_empty_and_blank() {
448 round_trips("");
449 round_trips("\n\n");
450 round_trips("# only a comment\n");
451 }
452
453 #[test]
454 fn produces_expected_top_level_shape() {
455 let parse = parse("func f():\n\tpass\n");
456 let root = parse.syntax_node();
457 assert_eq!(root.kind(), SyntaxKind::SourceFile);
458 let func = root
459 .children()
460 .find(|n| n.kind() == SyntaxKind::FuncDecl)
461 .expect("a FuncDecl child");
462 assert!(func.children().any(|n| n.kind() == SyntaxKind::Block));
463 }
464
465 fn contains_node(node: &ResolvedNode<SyntaxKind>, kind: SyntaxKind) -> bool {
466 node.kind() == kind || node.children().any(|c| contains_node(c, kind))
467 }
468
469 #[test]
470 fn property_accessors_parse_cleanly() {
471 let indented = "var x: int:\n\tget:\n\t\treturn 1\n\tset(v):\n\t\tx = v\n";
472 round_trips(indented);
473 assert!(
474 parse(indented).errors().is_empty(),
475 "valid get/set accessors must not error: {:?}",
476 parse(indented).errors()
477 );
478 let inline = "var y: int : get = _get_y, set = _set_y\n";
479 round_trips(inline);
480 assert!(
481 parse(inline).errors().is_empty(),
482 "{:?}",
483 parse(inline).errors()
484 );
485 }
486
487 #[test]
488 fn a_non_get_set_accessor_keyword_is_a_parse_error() {
489 let src = "var x: int:\n\tfoo:\n\t\treturn 1\n";
492 assert!(
493 !parse(src).errors().is_empty(),
494 "a non-get/set accessor must be a parse error"
495 );
496 }
497
498 fn has_match_stmt(src: &str) -> bool {
499 contains_node(&parse(src).syntax_node(), SyntaxKind::MatchStmt)
500 }
501
502 #[test]
503 fn a_real_match_statement_still_parses() {
504 let src = "func f():\n\tmatch x:\n\t\t1:\n\t\t\tpass\n";
505 round_trips(src);
506 assert!(has_match_stmt(src));
507 }
508
509 #[test]
510 fn a_parenthesised_match_subject_is_still_a_statement() {
511 let src = "func f():\n\tmatch (x):\n\t\t_:\n\t\t\tpass\n";
512 round_trips(src);
513 assert!(has_match_stmt(src), "`match (x):` is a match statement");
514 }
515
516 #[test]
517 fn match_used_as_an_identifier_is_an_expression_not_a_statement() {
518 for src in [
521 "func f():\n\tmatch.foo()\n",
522 "func f():\n\tmatch = 5\n",
523 "func f():\n\tmatch += 1\n",
524 "func f():\n\tmatch(x)\n",
525 "func f():\n\tmatch[0] = 1\n",
526 ] {
527 round_trips(src);
528 assert!(
529 !has_match_stmt(src),
530 "should be an expression, not a match statement: {src:?}"
531 );
532 }
533 }
534
535 fn node_sexpr(node: &ResolvedNode<SyntaxKind>) -> String {
538 let mut s = format!("({:?}", node.kind());
539 for child in node.children() {
540 s.push(' ');
541 s.push_str(&node_sexpr(child));
542 }
543 s.push(')');
544 s
545 }
546
547 fn structure(src: &str) -> String {
548 node_sexpr(&parse(src).syntax_node())
549 }
550
551 #[test]
552 fn precedence_factor_binds_tighter_than_add() {
553 assert_eq!(
555 structure("var x = 1 + 2 * 3\n"),
556 "(SourceFile (VarDecl (Name) (BinExpr (Literal) (BinExpr (Literal) (Literal)))))"
557 );
558 assert_eq!(
560 structure("var x = 1 * 2 + 3\n"),
561 "(SourceFile (VarDecl (Name) (BinExpr (BinExpr (Literal) (Literal)) (Literal))))"
562 );
563 }
564
565 #[test]
566 fn power_is_left_associative() {
567 assert_eq!(
569 structure("var x = 2 ** 3 ** 4\n"),
570 "(SourceFile (VarDecl (Name) (BinExpr (BinExpr (Literal) (Literal)) (Literal))))"
571 );
572 }
573
574 #[test]
575 fn unary_minus_then_power() {
576 assert_eq!(
578 structure("var x = -2 ** 2\n"),
579 "(SourceFile (VarDecl (Name) (UnaryExpr (BinExpr (Literal) (Literal)))))"
580 );
581 }
582
583 #[test]
584 fn ternary_is_right_associative() {
585 assert_eq!(
586 structure("var x = a if c else b\n"),
587 "(SourceFile (VarDecl (Name) (TernaryExpr (NameRef) (NameRef) (NameRef))))"
588 );
589 }
590
591 #[test]
592 fn postfix_chain_call_field_index() {
593 assert_eq!(
595 structure("var x = a.b().c[0]\n"),
596 "(SourceFile (VarDecl (Name) (IndexExpr (FieldExpr (CallExpr (FieldExpr (NameRef) \
597 (NameRef)) (ArgList)) (NameRef)) (Literal))))"
598 );
599 }
600
601 #[test]
602 fn leading_utf8_bom_is_trivia_not_an_error() {
603 let src = "\u{feff}class_name Foo\nextends Node\n";
606 let parse = parse(src);
607 assert_eq!(
608 parse.syntax_node().to_string(),
609 src,
610 "BOM file must round-trip byte-for-byte"
611 );
612 assert!(
613 parse.errors().is_empty(),
614 "BOM-prefixed file should parse clean: {:?}",
615 parse.errors()
616 );
617 assert!(
619 structure(src).starts_with("(SourceFile (ClassNameDecl"),
620 "{}",
621 structure(src)
622 );
623 }
624
625 #[test]
626 fn multiline_lambda_does_not_absorb_following_paren_line() {
627 let src = "func f():\n\tvar cb := func():\n\t\treturn 1\n\t(self).process()\n";
632 let st = structure(src);
633 assert!(
634 st.contains("(VarDecl (Name) (LambdaExpr"),
635 "lambda should be the var initializer, standalone: {st}"
636 );
637 assert!(
638 !st.contains("CallExpr (LambdaExpr"),
639 "the following `(` line must not be absorbed as a call on the lambda: {st}"
640 );
641 assert!(
643 st.contains("(ExprStmt (CallExpr (FieldExpr (ParenExpr"),
644 "the `(self).process()` line should be its own statement: {st}"
645 );
646 round_trips(src);
647 }
648
649 #[test]
650 fn inline_lambda_still_chains_postfix() {
651 let src = "var x = (func(): return 1).call()\n";
654 let st = structure(src);
655 assert!(
656 st.contains("CallExpr (FieldExpr (ParenExpr (LambdaExpr"),
657 "inline lambda should still accept a postfix chain: {st}"
658 );
659 round_trips(src);
660 }
661
662 #[test]
663 fn statement_level_annotation_in_a_body_parses_clean() {
664 let src = "func f():\n\t@warning_ignore(\"integer_division\")\n\tvar x := 1 / 2\n";
668 let parse = parse(src);
669 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
670 round_trips(src);
671 }
672
673 #[test]
674 fn multiline_lambda_arg_with_dedented_closer_parses_clean() {
675 let src = "func f():\n\tobj.call(func():\n\t\t\tbody()\n\t\t)\n";
679 let parse = parse(src);
680 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
681 round_trips(src);
682 }
683
684 #[test]
685 fn multiline_lambda_body_ending_at_a_comma_parses_clean() {
686 let src = "func f():\n\tobj.call(\n\t\tfunc(v):\n\t\t\tuse(v), 0.0, 1.0)\n";
690 let parse = parse(src);
691 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
692 round_trips(src);
693 }
694
695 const CORPUS: &str = r#"@tool
698class_name Player extends CharacterBody2D
699## A documented player controller.
700
701const SPEED := 300.0
702@export var health: int = 100
703@export_range(0, 100) var armor := 0
704static var instances: Array[Player] = []
705
706enum State { IDLE, RUNNING, JUMPING = 10 }
707
708signal died(reason: String)
709
710var _vel: Vector2 = Vector2.ZERO:
711 get:
712 return _vel
713 set(value):
714 _vel = value
715
716class Inner extends RefCounted:
717 var x = 1
718 func helper() -> int:
719 return x * 2
720
721func _ready() -> void:
722 var node := $Sprite2D
723 var unique = %HealthBar
724 add_child(preload("res://thing.tscn").instantiate())
725 for i in range(0, 10):
726 if i % 2 == 0 and i > 0:
727 print(i, " even")
728 elif i == 5:
729 continue
730 else:
731 pass
732 while health > 0:
733 health -= 1
734 match State.IDLE:
735 State.IDLE, State.RUNNING:
736 pass
737 [var first, ..]:
738 print(first)
739 {"key": var v} when v > 0:
740 print(v)
741 _:
742 breakpoint
743 var cb := func(a: int, b := 2) -> int: return a + b
744 var ok = node is Node2D
745 var cast = node as Sprite2D
746 assert(health >= 0, "negative health")
747 died.emit("test")
748"#;
749
750 #[test]
751 fn corpus_round_trips_byte_for_byte() {
752 round_trips(CORPUS);
753 }
754
755 #[test]
756 fn corpus_parses_without_unexpected_errors() {
757 let parse = parse(CORPUS);
759 assert!(
760 parse.errors().is_empty(),
761 "unexpected parse errors:\n{:#?}",
762 parse.errors()
763 );
764 }
765
766 #[test]
767 fn inline_if_elif_else_clauses_attach() {
768 let src = "func f():\n\tif a: x = 1\n\telif b: x = 2\n\telse: x = 3\n";
773 let parse = parse(src);
774 assert_eq!(parse.syntax_node().to_string(), src, "lossless");
775 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
776 let root = parse.syntax_node();
777 let if_stmt = root
778 .descendants()
779 .find(|n| n.kind() == SyntaxKind::IfStmt)
780 .expect("an IfStmt node");
781 assert!(
782 if_stmt
783 .descendants()
784 .any(|n| n.kind() == SyntaxKind::ElifClause),
785 "elif clause attached to the if"
786 );
787 assert!(
788 if_stmt
789 .descendants()
790 .any(|n| n.kind() == SyntaxKind::ElseClause),
791 "else clause attached to the if"
792 );
793 }
794
795 #[test]
796 fn soft_keyword_names_parse() {
797 let src = "static func match(when: bool) -> int:\n\tvar r = RUIRouteMatcher.match(when)\n\treturn when\n";
801 let parse = parse(src);
802 assert_eq!(parse.syntax_node().to_string(), src, "lossless");
803 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
804 }
805
806 #[test]
807 fn multiline_lambda_with_trailing_call_paren() {
808 let src = "func f():\n\tt.connect(func():\n\t\tif ok:\n\t\t\tp.free())\n";
811 let parse = parse(src);
812 assert_eq!(parse.syntax_node().to_string(), src, "lossless");
813 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
814 }
815
816 #[test]
817 fn multiline_lambda_in_call_argument_parses() {
818 let src = "func f():\n\tcb(func(a, b):\n\t\treturn a + b\n\t)\n";
820 let parse = parse(src);
821 assert_eq!(parse.syntax_node().to_string(), src, "lossless");
822 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
823 let root = parse.syntax_node();
824 let lambda = root
825 .descendants()
826 .find(|n| n.kind() == SyntaxKind::LambdaExpr)
827 .expect("a LambdaExpr node");
828 let block = lambda
829 .children()
830 .find(|n| n.kind() == SyntaxKind::Block)
831 .expect("the lambda body Block");
832 assert!(
833 block
834 .descendants()
835 .any(|n| n.kind() == SyntaxKind::ReturnStmt),
836 "the lambda body contains the return statement"
837 );
838 }
839
840 #[test]
841 fn single_line_lambda_in_call_argument_parses() {
842 let src = "var m = arr.map(func(x): x * 2)\n";
844 let parse = parse(src);
845 assert_eq!(parse.syntax_node().to_string(), src, "lossless");
846 assert!(parse.errors().is_empty(), "no errors: {:?}", parse.errors());
847 assert!(
848 parse
849 .syntax_node()
850 .descendants()
851 .any(|n| n.kind() == SyntaxKind::LambdaExpr),
852 "a LambdaExpr node"
853 );
854 }
855
856 #[test]
857 fn broken_code_recovers_and_round_trips() {
858 let src = "func ok():\n\tpass\nfunc bad(:\n\tpass\nfunc also_ok():\n\tpass\n";
861 let parse = parse(src);
862 assert_eq!(
863 parse.syntax_node().to_string(),
864 src,
865 "recovery must stay lossless"
866 );
867 assert!(!parse.errors().is_empty(), "expected a syntax error");
868 let funcs = parse
870 .syntax_node()
871 .children()
872 .filter(|n| n.kind() == SyntaxKind::FuncDecl)
873 .count();
874 assert!(
875 funcs >= 2,
876 "siblings should survive a broken declaration, got {funcs}"
877 );
878 }
879
880 #[test]
881 fn over_indented_body_line_does_not_cascade_to_class_level() {
882 let src =
889 "func render():\n\tvar a = 1\n\t\tvar bad = 2\n\tvar b = 2\n\tfor i in 3:\n\t\tpass\n";
890 let parse = parse(src);
891 assert_eq!(
892 parse.syntax_node().to_string(),
893 src,
894 "recovery stays lossless"
895 );
896 assert_eq!(
897 parse.errors().len(),
898 1,
899 "exactly one diagnostic, not a cascade: {:?}",
900 parse.errors()
901 );
902 let root = parse.syntax_node();
904 let top_funcs = root
905 .children()
906 .filter(|n| n.kind() == SyntaxKind::FuncDecl)
907 .count();
908 assert_eq!(top_funcs, 1, "the whole body stays in one function");
909 assert!(
910 !root.children().any(|n| n.kind() == SyntaxKind::VarDecl),
911 "no body statement leaked to class level: {}",
912 parse.debug_tree()
913 );
914 assert!(
916 root.descendants().any(|n| n.kind() == SyntaxKind::ForStmt),
917 "the trailing `for` is recovered as a body statement"
918 );
919 }
920
921 #[test]
922 fn golden_small_class() {
923 let parse = parse("class_name Foo\nvar x := 1\n");
924 expect_test::expect_file!["../test_data/golden/small_class.cst"]
925 .assert_eq(&parse.debug_tree());
926 }
927}