brokk_tree_sitter_typescript/
lib.rs

1//! This crate provides TypeScript and TSX language support for the [tree-sitter][] parsing library.
2//!
3//! Typically, you will use the [LANGUAGE_TYPESCRIPT] constant to add this language to a
4//! tree-sitter [Parser][], and then use the parser to parse some code:
5//!
6//! ```
7//! use tree_sitter::Parser;
8//!
9//! let code = r#"
10//! function double(x: number): number {
11//!     return x * 2;
12//! }
13//! "#;
14//! let mut parser = Parser::new();
15//! let language = brokk_tree_sitter_typescript::LANGUAGE_TYPESCRIPT;
16//! parser
17//!     .set_language(&language.into())
18//!     .expect("Error loading TypeScript parser");
19//! let tree = parser.parse(code, None).unwrap();
20//! assert!(!tree.root_node().has_error());
21//! ```
22//!
23//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
24//! [tree-sitter]: https://tree-sitter.github.io/
25
26use tree_sitter_language::LanguageFn;
27
28extern "C" {
29    fn brokk_tree_sitter_typescript() -> *const ();
30    fn brokk_tree_sitter_tsx() -> *const ();
31}
32
33/// The tree-sitter [`LanguageFn`] for TypeScript.
34///
35/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
36pub const LANGUAGE_TYPESCRIPT: LanguageFn =
37    unsafe { LanguageFn::from_raw(brokk_tree_sitter_typescript) };
38
39/// The tree-sitter [`LanguageFn`] for TSX.
40///
41/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html
42pub const LANGUAGE_TSX: LanguageFn = unsafe { LanguageFn::from_raw(brokk_tree_sitter_tsx) };
43
44/// The content of the [`node-types.json`][] file for TypeScript.
45///
46/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
47pub const TYPESCRIPT_NODE_TYPES: &str = include_str!("../../typescript/src/node-types.json");
48
49/// The content of the [`node-types.json`][] file for TSX.
50///
51/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
52pub const TSX_NODE_TYPES: &str = include_str!("../../tsx/src/node-types.json");
53
54/// The syntax highlighting query for TypeScript.
55pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
56
57/// The local-variable syntax highlighting query for TypeScript.
58pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
59
60/// The symbol tagging query for TypeScript.
61pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
62
63#[cfg(test)]
64mod tests {
65    fn parse_bytes(language: tree_sitter_language::LanguageFn, source: &[u8]) -> tree_sitter::Tree {
66        let mut parser = tree_sitter::Parser::new();
67        parser.set_language(&language.into()).unwrap();
68        parser.parse(source, None).unwrap()
69    }
70
71    fn has_node_bytes(
72        node: tree_sitter::Node<'_>,
73        source: &[u8],
74        kind: &str,
75        expected: &[u8],
76    ) -> bool {
77        if node.kind() == kind && source.get(node.start_byte()..node.end_byte()) == Some(expected) {
78            return true;
79        }
80
81        (0..node.child_count())
82            .any(|index| has_node_bytes(node.child(index).unwrap(), source, kind, expected))
83    }
84
85    fn count_nodes(node: tree_sitter::Node<'_>, kind: &str) -> usize {
86        usize::from(node.kind() == kind)
87            + (0..node.child_count())
88                .map(|index| count_nodes(node.child(index).unwrap(), kind))
89                .sum::<usize>()
90    }
91
92    fn assert_complete_source(tree: &tree_sitter::Tree, source: &[u8]) {
93        let root = tree.root_node();
94        assert_eq!(root.start_byte(), 0);
95        assert_eq!(root.end_byte(), source.len());
96        assert_eq!(root.utf8_text(source).unwrap().as_bytes(), source);
97    }
98
99    fn assert_node_bytes(tree: &tree_sitter::Tree, source: &[u8], kind: &str, expected: &[u8]) {
100        assert!(
101            has_node_bytes(tree.root_node(), source, kind, expected),
102            "missing {kind} node with exact bytes {expected:?}: {}",
103            tree.root_node().to_sexp()
104        );
105    }
106
107    #[test]
108    fn test_can_load_typescript_grammar() {
109        let mut parser = tree_sitter::Parser::new();
110        parser
111            .set_language(&super::LANGUAGE_TYPESCRIPT.into())
112            .expect("Error loading TypeScript parser");
113    }
114
115    #[test]
116    fn test_can_load_tsx_grammar() {
117        let mut parser = tree_sitter::Parser::new();
118        parser
119            .set_language(&super::LANGUAGE_TSX.into())
120            .expect("Error loading TSX parser");
121    }
122
123    #[test]
124    fn test_can_coexist_with_upstream_grammars() {
125        let mut parser = tree_sitter::Parser::new();
126        for language in [
127            super::LANGUAGE_TYPESCRIPT,
128            super::LANGUAGE_TSX,
129            tree_sitter_typescript::LANGUAGE_TYPESCRIPT,
130            tree_sitter_typescript::LANGUAGE_TSX,
131        ] {
132            parser
133                .set_language(&language.into())
134                .expect("grammar must coexist with its upstream counterpart");
135            let tree = parser.parse("const pattern = /value+/;", None).unwrap();
136            assert!(!tree.root_node().has_error());
137        }
138        assert_ne!(
139            tree_sitter::Language::from(super::LANGUAGE_TYPESCRIPT),
140            tree_sitter::Language::from(tree_sitter_typescript::LANGUAGE_TYPESCRIPT)
141        );
142        assert_ne!(
143            tree_sitter::Language::from(super::LANGUAGE_TSX),
144            tree_sitter::Language::from(tree_sitter_typescript::LANGUAGE_TSX)
145        );
146    }
147
148    #[test]
149    fn test_declaration_forms_in_both_dialects() {
150        let source = r#"
151type Ref<T> = import("widgets").Refs.Element<T>;
152interface Options {
153  previous?: Ref<HTMLElement>
154  // Reserved words can name type members after a newline.
155  in?: string[]
156  instanceof: boolean
157  abstract?: string
158  readonly?: boolean
159}
160declare abstract class Base {
161  abstract readonly kind: string;
162}
163declare abstract class Derived extends Base {
164  public abstract override readonly kind: "number" | "bigint";
165  tail: string;
166}
167"#;
168        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
169            let mut parser = tree_sitter::Parser::new();
170            parser.set_language(&language.into()).unwrap();
171            let tree = parser.parse(source, None).unwrap();
172            assert!(
173                !tree.root_node().has_error(),
174                "{}",
175                tree.root_node().to_sexp()
176            );
177        }
178    }
179
180    #[test]
181    fn test_import_type_compositions_in_both_dialects() {
182        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
183            let mut parser = tree_sitter::Parser::new();
184            parser.set_language(&language.into()).unwrap();
185            for (source, kind) in [
186                (r#"type ItemList = import("./m").Thing[];"#, "array_type"),
187                (
188                    r#"type Member = import("./m").Container["member"];"#,
189                    "lookup_type",
190                ),
191                (
192                    r#"type Nested = import("./m").NS.Box<string>["items"][][];"#,
193                    "array_type",
194                ),
195                (r#"type Module = import("./m")["Thing"];"#, "lookup_type"),
196                (
197                    r#"type Keys = keyof import("./m").Thing;"#,
198                    "index_type_query",
199                ),
200                (
201                    r#"type ReadonlyList = readonly import("./m").Thing[];"#,
202                    "readonly_type",
203                ),
204                (
205                    r#"type Union = import("./m").Thing | undefined;"#,
206                    "union_type",
207                ),
208                (
209                    r#"type Intersection = import("./m").Thing & { id: string };"#,
210                    "intersection_type",
211                ),
212            ] {
213                let tree = parser.parse(source, None).unwrap();
214                assert!(
215                    !tree.root_node().has_error(),
216                    "{source}: {}",
217                    tree.root_node().to_sexp()
218                );
219                let alias = tree.root_node().named_child(0).unwrap();
220                assert_eq!(
221                    alias.child_by_field_name("value").unwrap().kind(),
222                    kind,
223                    "{source}"
224                );
225            }
226        }
227    }
228
229    #[test]
230    fn test_tsx_literal_ampersands_and_entities() {
231        fn entity_count(node: tree_sitter::Node) -> usize {
232            let mut cursor = node.walk();
233            usize::from(node.kind() == "html_character_reference")
234                + node
235                    .named_children(&mut cursor)
236                    .map(entity_count)
237                    .sum::<usize>()
238        }
239
240        let mut parser = tree_sitter::Parser::new();
241        parser.set_language(&super::LANGUAGE_TSX.into()).unwrap();
242        for (source, entities) in [
243            (
244                r#"const x = <p a="&" b='&' c="x&" d='x&'>Roles & Groups</p>;"#,
245                0,
246            ),
247            (
248                r#"const x = <p a="?a=1&b=2" b='?a=1&b=2'>&foo &; &#bad &</p>;"#,
249                0,
250            ),
251            (
252                r#"const x = <p a="&amp;&#38;&#x26;" b='&amp;&#38;&#x26;'>&amp;&#38;&#x26;</p>;"#,
253                9,
254            ),
255            (
256                r#"const x = <><p a="&" />&amp;&{value}<span>&</span></>;"#,
257                1,
258            ),
259        ] {
260            let tree = parser.parse(source, None).unwrap();
261            assert!(
262                !tree.root_node().has_error(),
263                "{source}: {}",
264                tree.root_node().to_sexp()
265            );
266            assert_eq!(entity_count(tree.root_node()), entities, "{source}");
267        }
268        for source in [
269            "const x = <p a=\"&>text</p>;",
270            "const x = <p a='&>text</p>;",
271            "const x = <p>&;",
272        ] {
273            let tree = parser.parse(source, None).unwrap();
274            assert!(
275                tree.root_node().has_error(),
276                "unexpectedly accepted: {source}"
277            );
278        }
279    }
280
281    #[test]
282    fn test_expression_and_malformed_near_misses() {
283        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
284            let mut parser = tree_sitter::Parser::new();
285            parser.set_language(&language.into()).unwrap();
286            for (source, operator) in [
287                ("const found = key\n in record;", "in"),
288                ("const found = item\n instanceof Widget;", "instanceof"),
289            ] {
290                let tree = parser.parse(source, None).unwrap();
291                assert!(!tree.root_node().has_error());
292                let value = tree
293                    .root_node()
294                    .named_child(0)
295                    .unwrap()
296                    .named_child(0)
297                    .unwrap()
298                    .child_by_field_name("value")
299                    .unwrap();
300                assert_eq!(value.kind(), "binary_expression");
301                assert_eq!(
302                    value
303                        .child_by_field_name("operator")
304                        .unwrap()
305                        .utf8_text(source.as_bytes())
306                        .unwrap(),
307                    operator
308                );
309            }
310            for source in [
311                "type Broken = { previous: string in?: string[] };",
312                "type Broken = import(\"widgets\").Ref<;",
313                "abstract class Broken { abstract override readonly value: ; }",
314            ] {
315                let tree = parser.parse(source, None).unwrap();
316                assert!(
317                    tree.root_node().has_error(),
318                    "unexpectedly accepted: {source}"
319                );
320            }
321        }
322    }
323
324    #[test]
325    fn test_raw_nul_content_is_preserved_in_both_dialects() {
326        // NUL in a quoted module specifier is still string content. This checks
327        // syntax parsing only; it makes no claim about resolving that module.
328        let cases: &[(&[u8], &str, &[u8])] = &[
329            (
330                b"const double_raw = \"before\0after\";\nconst after = 1;",
331                "string",
332                b"\"before\0after\"",
333            ),
334            (
335                b"const single_raw = 'before\0after';\nconst after = 1;",
336                "string",
337                b"'before\0after'",
338            ),
339            (
340                b"const double_escape = \"before\\0after\";\nconst after = 1;",
341                "string",
342                b"\"before\\0after\"",
343            ),
344            (
345                b"const single_escape = 'before\\0after';\nconst after = 1;",
346                "string",
347                b"'before\\0after'",
348            ),
349            (
350                b"const slash_nul = \"before\\\0after\";\nconst after = 1;",
351                "escape_sequence",
352                b"\\\0",
353            ),
354            (
355                b"const adjacent = \"\0\0\";\nconst after = 1;",
356                "string",
357                b"\"\0\0\"",
358            ),
359            (
360                b"const only_nul = '\0';\nconst after = 1;",
361                "string",
362                b"'\0'",
363            ),
364            (
365                b"// before\0after\nconst after = 1;",
366                "comment",
367                b"// before\0after",
368            ),
369            (b"//\0\nconst after = 1;", "comment", b"//\0"),
370            (
371                b"const block = 1; /* before\0after */\nconst after = 1;",
372                "comment",
373                b"/* before\0after */",
374            ),
375            (b"/*\0*/\nconst after = 1;", "comment", b"/*\0*/"),
376            (b"<!--\0-->\nconst after = 1;", "html_comment", b"<!--\0-->"),
377            (
378                b"const template = `before\0after`;\nconst after = 1;",
379                "template_string",
380                b"`before\0after`",
381            ),
382            (
383                b"const only_nul_template = `\0`;\nconst after = 1;",
384                "template_string",
385                b"`\0`",
386            ),
387            (
388                b"const template = `before\0${value}after`;\nconst after = 1;",
389                "string_fragment",
390                b"before\0",
391            ),
392            (
393                b"type Pattern = `before\0${string}after`;\nconst after = 1;",
394                "template_literal_type",
395                b"`before\0${string}after`",
396            ),
397            (
398                b"type Imported = import(\"pkg\0suffix\").Member;\nconst after = 1;",
399                "string",
400                b"\"pkg\0suffix\"",
401            ),
402        ];
403
404        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
405            for (source, kind, raw_node) in cases {
406                let tree = parse_bytes(language, source);
407                assert!(
408                    !tree.root_node().has_error(),
409                    "unexpected parse error for {source:?}: {}",
410                    tree.root_node().to_sexp()
411                );
412                assert_complete_source(&tree, source);
413                assert_node_bytes(&tree, source, kind, raw_node);
414                assert_node_bytes(&tree, source, "lexical_declaration", b"const after = 1;");
415
416                if source.starts_with(b"const template = `before\0${") {
417                    assert_eq!(count_nodes(tree.root_node(), "template_substitution"), 1);
418                }
419            }
420        }
421
422        let line_comment_at_eof = b"// before\0";
423        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
424            let tree = parse_bytes(language, line_comment_at_eof);
425            assert!(
426                !tree.root_node().has_error(),
427                "{}",
428                tree.root_node().to_sexp()
429            );
430            assert_complete_source(&tree, line_comment_at_eof);
431            assert_node_bytes(&tree, line_comment_at_eof, "comment", line_comment_at_eof);
432        }
433    }
434
435    #[test]
436    fn test_raw_nul_is_preserved_in_tsx_attributes_and_text() {
437        let cases: &[(&[u8], &str, &[u8])] = &[
438            (
439                b"const view = <Widget title=\"before\0after\" />;\nconst after = 1;",
440                "string",
441                b"\"before\0after\"",
442            ),
443            (
444                b"const view = <p>before\0after</p>;\nconst after = 1;",
445                "jsx_text",
446                b"before\0after",
447            ),
448            (
449                b"const view = <Widget title=\"&\0x\" />;\nconst after = 1;",
450                "string",
451                b"\"&\0x\"",
452            ),
453            (
454                b"const view = <Widget title='&\0' />;\nconst after = 1;",
455                "string",
456                b"'&\0'",
457            ),
458        ];
459
460        for (source, kind, raw_node) in cases {
461            let tree = parse_bytes(super::LANGUAGE_TSX, source);
462            assert!(
463                !tree.root_node().has_error(),
464                "unexpected TSX parse error for {source:?}: {}",
465                tree.root_node().to_sexp()
466            );
467            assert_complete_source(&tree, source);
468            assert_node_bytes(&tree, source, kind, raw_node);
469            assert_node_bytes(&tree, source, "lexical_declaration", b"const after = 1;");
470            assert_eq!(count_nodes(tree.root_node(), "html_character_reference"), 0);
471        }
472    }
473
474    #[test]
475    fn test_raw_nul_in_executable_syntax_is_an_error_in_both_dialects() {
476        let cases: &[&[u8]] = &[
477            b"const na\0me = 1;\nconst after = 2;",
478            b"const result = left\0+ right;\nconst after = 2;",
479            b"const result = `raw\0${left\0+right}`;\nconst after = 2;",
480            b"type Module = import\0(\"pkg\").Member;\nconst after = 2;",
481        ];
482
483        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
484            for source in cases {
485                let tree = parse_bytes(language, source);
486                assert!(
487                    tree.root_node().has_error(),
488                    "unexpectedly accepted executable NUL in {source:?}: {}",
489                    tree.root_node().to_sexp()
490                );
491                assert_complete_source(&tree, source);
492                assert_node_bytes(&tree, source, "lexical_declaration", b"const after = 2;");
493            }
494        }
495    }
496
497    #[test]
498    fn test_unterminated_raw_nul_content_errors_at_eof() {
499        let cases: &[&[u8]] = &[
500            b"const broken = \"before\0after",
501            b"const broken = 'before\0after",
502            b"const broken = 1; /* before\0after",
503            b"const broken = `before\0after",
504        ];
505
506        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
507            for source in cases {
508                let tree = parse_bytes(language, source);
509                assert!(
510                    tree.root_node().has_error(),
511                    "unterminated raw-NUL content unexpectedly parsed: {source:?}: {}",
512                    tree.root_node().to_sexp()
513                );
514                assert_complete_source(&tree, source);
515            }
516        }
517    }
518
519    #[test]
520    fn test_incremental_raw_nul_edits_match_fresh_parses_in_both_dialects() {
521        let content_cases: &[(&[u8], &[u8], &[u8])] = &[
522            (b"const text = \"A\";\nconst tail = 1;", b"\"\0\"", b"\"A\""),
523            (b"/*A*/\nconst tail = 1;", b"/*\0*/", b"/*A*/"),
524            (b"const text = `A${value}`;\nconst tail = 1;", b"\0", b"A"),
525        ];
526
527        for language in [super::LANGUAGE_TYPESCRIPT, super::LANGUAGE_TSX] {
528            for (source, nul_node, original_node) in content_cases {
529                let mut parser = tree_sitter::Parser::new();
530                parser.set_language(&language.into()).unwrap();
531                let offset = source.iter().position(|byte| *byte == b'A').unwrap();
532                let old_tree = parser.parse(*source, None).unwrap();
533                assert!(!old_tree.root_node().has_error());
534
535                let mut with_nul = source.to_vec();
536                with_nul[offset] = 0;
537                let incremental =
538                    parse_after_single_byte_edit(&mut parser, &old_tree, &with_nul, offset);
539                let fresh = parser.parse(with_nul.as_slice(), None).unwrap();
540                assert!(!incremental.root_node().has_error());
541                assert_complete_source(&incremental, &with_nul);
542                assert_incremental_tree_matches_fresh(&incremental, &fresh, &with_nul);
543                let node_kind = if source.starts_with(b"/*") {
544                    "comment"
545                } else if source.starts_with(b"const text = `") {
546                    "string_fragment"
547                } else {
548                    "string"
549                };
550                assert_node_bytes(&incremental, &with_nul, node_kind, nul_node);
551
552                let mut restored = with_nul.clone();
553                restored[offset] = b'A';
554                let incremental =
555                    parse_after_single_byte_edit(&mut parser, &incremental, &restored, offset);
556                let fresh = parser.parse(restored.as_slice(), None).unwrap();
557                assert!(!incremental.root_node().has_error());
558                assert_complete_source(&incremental, &restored);
559                assert_incremental_tree_matches_fresh(&incremental, &fresh, &restored);
560                assert_node_bytes(&incremental, &restored, node_kind, original_node);
561            }
562
563            let identifier_source = b"const name = 1;\nconst tail = 1;";
564            let mut parser = tree_sitter::Parser::new();
565            parser.set_language(&language.into()).unwrap();
566            let offset = identifier_source
567                .iter()
568                .position(|byte| *byte == b'm')
569                .unwrap();
570            let old_tree = parser.parse(identifier_source.as_slice(), None).unwrap();
571            assert!(!old_tree.root_node().has_error());
572            let mut invalid_identifier = identifier_source.to_vec();
573            invalid_identifier[offset] = 0;
574            let incremental =
575                parse_after_single_byte_edit(&mut parser, &old_tree, &invalid_identifier, offset);
576            let fresh = parser.parse(invalid_identifier.as_slice(), None).unwrap();
577            assert!(incremental.root_node().has_error());
578            assert_complete_source(&incremental, &invalid_identifier);
579            assert_incremental_tree_matches_fresh(&incremental, &fresh, &invalid_identifier);
580            assert_node_bytes(
581                &incremental,
582                &invalid_identifier,
583                "lexical_declaration",
584                b"const tail = 1;",
585            );
586        }
587    }
588
589    fn parse_after_single_byte_edit(
590        parser: &mut tree_sitter::Parser,
591        old_tree: &tree_sitter::Tree,
592        new_source: &[u8],
593        offset: usize,
594    ) -> tree_sitter::Tree {
595        let mut edited_tree = old_tree.clone();
596        edited_tree.edit(&tree_sitter::InputEdit {
597            start_byte: offset,
598            old_end_byte: offset + 1,
599            new_end_byte: offset + 1,
600            start_position: tree_sitter::Point {
601                row: 0,
602                column: offset,
603            },
604            old_end_position: tree_sitter::Point {
605                row: 0,
606                column: offset + 1,
607            },
608            new_end_position: tree_sitter::Point {
609                row: 0,
610                column: offset + 1,
611            },
612        });
613        parser.parse(new_source, Some(&edited_tree)).unwrap()
614    }
615
616    fn assert_incremental_tree_matches_fresh(
617        incremental: &tree_sitter::Tree,
618        fresh: &tree_sitter::Tree,
619        source: &[u8],
620    ) {
621        let incremental_root = incremental.root_node();
622        let fresh_root = fresh.root_node();
623        assert_eq!(incremental_root.to_sexp(), fresh_root.to_sexp());
624        assert_eq!(incremental_root.start_byte(), fresh_root.start_byte());
625        assert_eq!(incremental_root.end_byte(), fresh_root.end_byte());
626        assert_eq!(incremental_root.has_error(), fresh_root.has_error());
627        assert_eq!(
628            incremental_root.utf8_text(source).unwrap(),
629            fresh_root.utf8_text(source).unwrap()
630        );
631    }
632}
brokk_tree_sitter_typescript/lib.rs

brokk_tree_sitter_typescript/
lib.rs
