ra_ap_syntax/
lib.rs

1//! Syntax Tree library used throughout the rust-analyzer.
2//!
3//! Properties:
4//!   - easy and fast incremental re-parsing
5//!   - graceful handling of errors
6//!   - full-fidelity representation (*any* text can be precisely represented as
7//!     a syntax tree)
8//!
9//! For more information, see the [RFC]. Current implementation is inspired by
10//! the [Swift] one.
11//!
12//! The most interesting modules here are `syntax_node` (which defines concrete
13//! syntax tree) and [`ast`] (which defines abstract syntax tree on top of the
14//! CST). The actual parser live in a separate [`parser`] crate, though the
15//! lexer lives in this crate.
16//!
17//! See `api_walkthrough` test in this file for a quick API tour!
18//!
19//! [RFC]: <https://github.com/rust-lang/rfcs/pull/2256>
20//! [Swift]: <https://github.com/apple/swift/blob/13d593df6f359d0cb2fc81cfaac273297c539455/lib/Syntax/README.md>
21
22#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
23
24#[cfg(not(feature = "in-rust-tree"))]
25extern crate ra_ap_rustc_lexer as rustc_lexer;
26#[cfg(feature = "in-rust-tree")]
27extern crate rustc_driver as _;
28#[cfg(feature = "in-rust-tree")]
29extern crate rustc_lexer;
30
31mod parsing;
32mod ptr;
33mod syntax_error;
34mod syntax_node;
35#[cfg(test)]
36mod tests;
37mod validation;
38
39pub mod algo;
40pub mod ast;
41#[doc(hidden)]
42pub mod fuzz;
43pub mod hacks;
44pub mod syntax_editor;
45pub mod utils;
46
47use std::{marker::PhantomData, ops::Range};
48
49use stdx::format_to;
50use triomphe::Arc;
51
52pub use crate::{
53    ast::{AstNode, AstToken},
54    ptr::{AstPtr, SyntaxNodePtr},
55    syntax_error::SyntaxError,
56    syntax_node::{
57        PreorderWithTokens, RustLanguage, SyntaxElement, SyntaxElementChildren, SyntaxNode,
58        SyntaxNodeChildren, SyntaxToken, SyntaxTreeBuilder,
59    },
60};
61pub use parser::{Edition, SyntaxKind, T};
62pub use rowan::{
63    Direction, GreenNode, NodeOrToken, SyntaxText, TextRange, TextSize, TokenAtOffset, WalkEvent,
64    api::Preorder,
65};
66pub use rustc_literal_escaper as unescape;
67pub use smol_str::{SmolStr, SmolStrBuilder, ToSmolStr, format_smolstr};
68
69/// `Parse` is the result of the parsing: a syntax tree and a collection of
70/// errors.
71///
72/// Note that we always produce a syntax tree, even for completely invalid
73/// files.
74#[derive(Debug, PartialEq, Eq)]
75pub struct Parse<T> {
76    green: Option<GreenNode>,
77    errors: Option<Arc<[SyntaxError]>>,
78    _ty: PhantomData<fn() -> T>,
79}
80
81impl<T> Clone for Parse<T> {
82    fn clone(&self) -> Parse<T> {
83        Parse { green: self.green.clone(), errors: self.errors.clone(), _ty: PhantomData }
84    }
85}
86
87impl<T> Parse<T> {
88    fn new(green: GreenNode, errors: Vec<SyntaxError>) -> Parse<T> {
89        Parse {
90            green: Some(green),
91            errors: if errors.is_empty() { None } else { Some(errors.into()) },
92            _ty: PhantomData,
93        }
94    }
95
96    pub fn syntax_node(&self) -> SyntaxNode {
97        SyntaxNode::new_root(self.green.as_ref().unwrap().clone())
98    }
99
100    pub fn errors(&self) -> Vec<SyntaxError> {
101        let mut errors = if let Some(e) = self.errors.as_deref() { e.to_vec() } else { vec![] };
102        validation::validate(&self.syntax_node(), &mut errors);
103        errors
104    }
105}
106
107impl<T: AstNode> Parse<T> {
108    /// Converts this parse result into a parse result for an untyped syntax tree.
109    pub fn to_syntax(mut self) -> Parse<SyntaxNode> {
110        let green = self.green.take();
111        let errors = self.errors.take();
112        Parse { green, errors, _ty: PhantomData }
113    }
114
115    /// Gets the parsed syntax tree as a typed ast node.
116    ///
117    /// # Panics
118    ///
119    /// Panics if the root node cannot be casted into the typed ast node
120    /// (e.g. if it's an `ERROR` node).
121    pub fn tree(&self) -> T {
122        T::cast(self.syntax_node()).unwrap()
123    }
124
125    /// Converts from `Parse<T>` to [`Result<T, Vec<SyntaxError>>`].
126    pub fn ok(self) -> Result<T, Vec<SyntaxError>> {
127        match self.errors() {
128            errors if !errors.is_empty() => Err(errors),
129            _ => Ok(self.tree()),
130        }
131    }
132}
133
134impl Parse<SyntaxNode> {
135    pub fn cast<N: AstNode>(mut self) -> Option<Parse<N>> {
136        if N::cast(self.syntax_node()).is_some() {
137            Some(Parse { green: self.green.take(), errors: self.errors.take(), _ty: PhantomData })
138        } else {
139            None
140        }
141    }
142}
143
144impl Parse<SourceFile> {
145    pub fn debug_dump(&self) -> String {
146        let mut buf = format!("{:#?}", self.tree().syntax());
147        for err in self.errors() {
148            format_to!(buf, "error {:?}: {}\n", err.range(), err);
149        }
150        buf
151    }
152
153    pub fn reparse(&self, delete: TextRange, insert: &str, edition: Edition) -> Parse<SourceFile> {
154        self.incremental_reparse(delete, insert, edition)
155            .unwrap_or_else(|| self.full_reparse(delete, insert, edition))
156    }
157
158    fn incremental_reparse(
159        &self,
160        delete: TextRange,
161        insert: &str,
162        edition: Edition,
163    ) -> Option<Parse<SourceFile>> {
164        // FIXME: validation errors are not handled here
165        parsing::incremental_reparse(
166            self.tree().syntax(),
167            delete,
168            insert,
169            self.errors.as_deref().unwrap_or_default().iter().cloned(),
170            edition,
171        )
172        .map(|(green_node, errors, _reparsed_range)| Parse {
173            green: Some(green_node),
174            errors: if errors.is_empty() { None } else { Some(errors.into()) },
175            _ty: PhantomData,
176        })
177    }
178
179    fn full_reparse(&self, delete: TextRange, insert: &str, edition: Edition) -> Parse<SourceFile> {
180        let mut text = self.tree().syntax().text().to_string();
181        text.replace_range(Range::<usize>::from(delete), insert);
182        SourceFile::parse(&text, edition)
183    }
184}
185
186impl ast::Expr {
187    /// Parses an `ast::Expr` from `text`.
188    ///
189    /// Note that if the parsed root node is not a valid expression, [`Parse::tree`] will panic.
190    /// For example:
191    /// ```rust,should_panic
192    /// # use syntax::{ast, Edition};
193    /// ast::Expr::parse("let fail = true;", Edition::CURRENT).tree();
194    /// ```
195    pub fn parse(text: &str, edition: Edition) -> Parse<ast::Expr> {
196        let _p = tracing::info_span!("Expr::parse").entered();
197        let (green, errors) = parsing::parse_text_at(text, parser::TopEntryPoint::Expr, edition);
198        let root = SyntaxNode::new_root(green.clone());
199
200        assert!(
201            ast::Expr::can_cast(root.kind()) || root.kind() == SyntaxKind::ERROR,
202            "{:?} isn't an expression",
203            root.kind()
204        );
205        Parse::new(green, errors)
206    }
207}
208
209#[cfg(not(no_salsa_async_drops))]
210impl<T> Drop for Parse<T> {
211    fn drop(&mut self) {
212        let Some(green) = self.green.take() else {
213            return;
214        };
215        static PARSE_DROP_THREAD: std::sync::OnceLock<std::sync::mpsc::Sender<GreenNode>> =
216            std::sync::OnceLock::new();
217        PARSE_DROP_THREAD
218            .get_or_init(|| {
219                let (sender, receiver) = std::sync::mpsc::channel::<GreenNode>();
220                std::thread::Builder::new()
221                    .name("ParseNodeDropper".to_owned())
222                    .spawn(move || {
223                        loop {
224                            // block on a receive
225                            _ = receiver.recv();
226                            // then drain the entire channel
227                            while receiver.try_recv().is_ok() {}
228                            // and sleep for a bit
229                            std::thread::sleep(std::time::Duration::from_millis(100));
230                        }
231                        // why do this over just a `receiver.iter().for_each(drop)`? To reduce contention on the channel lock.
232                        // otherwise this thread will constantly wake up and sleep again.
233                    })
234                    .unwrap();
235                sender
236            })
237            .send(green)
238            .unwrap();
239    }
240}
241
242/// `SourceFile` represents a parse tree for a single Rust file.
243pub use crate::ast::SourceFile;
244
245impl SourceFile {
246    pub fn parse(text: &str, edition: Edition) -> Parse<SourceFile> {
247        let _p = tracing::info_span!("SourceFile::parse").entered();
248        let (green, errors) = parsing::parse_text(text, edition);
249        let root = SyntaxNode::new_root(green.clone());
250
251        assert_eq!(root.kind(), SyntaxKind::SOURCE_FILE);
252        Parse::new(green, errors)
253    }
254}
255
256/// Matches a `SyntaxNode` against an `ast` type.
257///
258/// # Example:
259///
260/// ```ignore
261/// match_ast! {
262///     match node {
263///         ast::CallExpr(it) => { ... },
264///         ast::MethodCallExpr(it) => { ... },
265///         ast::MacroCall(it) => { ... },
266///         _ => None,
267///     }
268/// }
269/// ```
270#[macro_export]
271macro_rules! match_ast {
272    (match $node:ident { $($tt:tt)* }) => { $crate::match_ast!(match ($node) { $($tt)* }) };
273
274    (match ($node:expr) {
275        $( $( $path:ident )::+ ($it:pat) $(if $guard:expr)? => $res:expr, )*
276        _ => $catch_all:expr $(,)?
277    }) => {{
278        #[allow(clippy::question_mark, reason = "if `$catch_all` is `return None` Clippy can mark this")]
279        {
280            $( if let Some($it) = $($path::)+cast($node.clone()) $(&& $guard)? { $res } else )*
281            { $catch_all }
282        }
283    }};
284}
285
286/// This test does not assert anything and instead just shows off the crate's
287/// API.
288#[test]
289fn api_walkthrough() {
290    use ast::{HasModuleItem, HasName};
291
292    let source_code = "
293        fn foo() {
294            1 + 1
295        }
296    ";
297    // `SourceFile` is the main entry point.
298    //
299    // The `parse` method returns a `Parse` -- a pair of syntax tree and a list
300    // of errors. That is, syntax tree is constructed even in presence of errors.
301    let parse = SourceFile::parse(source_code, parser::Edition::CURRENT);
302    assert!(parse.errors().is_empty());
303
304    // The `tree` method returns an owned syntax node of type `SourceFile`.
305    // Owned nodes are cheap: inside, they are `Rc` handles to the underlying data.
306    let file: SourceFile = parse.tree();
307
308    // `SourceFile` is the root of the syntax tree. We can iterate file's items.
309    // Let's fetch the `foo` function.
310    let mut func = None;
311    for item in file.items() {
312        match item {
313            ast::Item::Fn(f) => func = Some(f),
314            _ => unreachable!(),
315        }
316    }
317    let func: ast::Fn = func.unwrap();
318
319    // Each AST node has a bunch of getters for children. All getters return
320    // `Option`s though, to account for incomplete code. Some getters are common
321    // for several kinds of node. In this case, a trait like `ast::NameOwner`
322    // usually exists. By convention, all ast types should be used with `ast::`
323    // qualifier.
324    let name: Option<ast::Name> = func.name();
325    let name = name.unwrap();
326    assert_eq!(name.text(), "foo");
327
328    // Let's get the `1 + 1` expression!
329    let body: ast::BlockExpr = func.body().unwrap();
330    let stmt_list: ast::StmtList = body.stmt_list().unwrap();
331    let expr: ast::Expr = stmt_list.tail_expr().unwrap();
332
333    // Enums are used to group related ast nodes together, and can be used for
334    // matching. However, because there are no public fields, it's possible to
335    // match only the top level enum: that is the price we pay for increased API
336    // flexibility
337    let bin_expr: &ast::BinExpr = match &expr {
338        ast::Expr::BinExpr(e) => e,
339        _ => unreachable!(),
340    };
341
342    // Besides the "typed" AST API, there's an untyped CST one as well.
343    // To switch from AST to CST, call `.syntax()` method:
344    let expr_syntax: &SyntaxNode = expr.syntax();
345
346    // Note how `expr` and `bin_expr` are in fact the same node underneath:
347    assert!(expr_syntax == bin_expr.syntax());
348
349    // To go from CST to AST, `AstNode::cast` function is used:
350    let _expr: ast::Expr = match ast::Expr::cast(expr_syntax.clone()) {
351        Some(e) => e,
352        None => unreachable!(),
353    };
354
355    // The two properties each syntax node has is a `SyntaxKind`:
356    assert_eq!(expr_syntax.kind(), SyntaxKind::BIN_EXPR);
357
358    // And text range:
359    assert_eq!(expr_syntax.text_range(), TextRange::new(32.into(), 37.into()));
360
361    // You can get node's text as a `SyntaxText` object, which will traverse the
362    // tree collecting token's text:
363    let text: SyntaxText = expr_syntax.text();
364    assert_eq!(text.to_string(), "1 + 1");
365
366    // There's a bunch of traversal methods on `SyntaxNode`:
367    assert_eq!(expr_syntax.parent().as_ref(), Some(stmt_list.syntax()));
368    assert_eq!(stmt_list.syntax().first_child_or_token().map(|it| it.kind()), Some(T!['{']));
369    assert_eq!(
370        expr_syntax.next_sibling_or_token().map(|it| it.kind()),
371        Some(SyntaxKind::WHITESPACE)
372    );
373
374    // As well as some iterator helpers:
375    let f = expr_syntax.ancestors().find_map(ast::Fn::cast);
376    assert_eq!(f, Some(func));
377    assert!(expr_syntax.siblings_with_tokens(Direction::Next).any(|it| it.kind() == T!['}']));
378    assert_eq!(
379        expr_syntax.descendants_with_tokens().count(),
380        8, // 5 tokens `1`, ` `, `+`, ` `, `1`
381           // 2 child literal expressions: `1`, `1`
382           // 1 the node itself: `1 + 1`
383    );
384
385    // There's also a `preorder` method with a more fine-grained iteration control:
386    let mut buf = String::new();
387    let mut indent = 0;
388    for event in expr_syntax.preorder_with_tokens() {
389        match event {
390            WalkEvent::Enter(node) => {
391                let text = match &node {
392                    NodeOrToken::Node(it) => it.text().to_string(),
393                    NodeOrToken::Token(it) => it.text().to_owned(),
394                };
395                format_to!(buf, "{:indent$}{:?} {:?}\n", " ", text, node.kind(), indent = indent);
396                indent += 2;
397            }
398            WalkEvent::Leave(_) => indent -= 2,
399        }
400    }
401    assert_eq!(indent, 0);
402    assert_eq!(
403        buf.trim(),
404        r#"
405"1 + 1" BIN_EXPR
406  "1" LITERAL
407    "1" INT_NUMBER
408  " " WHITESPACE
409  "+" PLUS
410  " " WHITESPACE
411  "1" LITERAL
412    "1" INT_NUMBER
413"#
414        .trim()
415    );
416
417    // To recursively process the tree, there are three approaches:
418    // 1. explicitly call getter methods on AST nodes.
419    // 2. use descendants and `AstNode::cast`.
420    // 3. use descendants and `match_ast!`.
421    //
422    // Here's how the first one looks like:
423    let exprs_cast: Vec<String> = file
424        .syntax()
425        .descendants()
426        .filter_map(ast::Expr::cast)
427        .map(|expr| expr.syntax().text().to_string())
428        .collect();
429
430    // An alternative is to use a macro.
431    let mut exprs_visit = Vec::new();
432    for node in file.syntax().descendants() {
433        match_ast! {
434            match node {
435                ast::Expr(it) => {
436                    let res = it.syntax().text().to_string();
437                    exprs_visit.push(res);
438                },
439                _ => (),
440            }
441        }
442    }
443    assert_eq!(exprs_cast, exprs_visit);
444}
ra_ap_syntax/lib.rs

ra_ap_syntax/
lib.rs
