ra_ap_syntax/
lib.rs

1//! Syntax Tree library used throughout the rust-analyzer.
2//!
3//! Properties:
4//!   - easy and fast incremental re-parsing
5//!   - graceful handling of errors
6//!   - full-fidelity representation (*any* text can be precisely represented as
7//!     a syntax tree)
8//!
9//! For more information, see the [RFC]. Current implementation is inspired by
10//! the [Swift] one.
11//!
12//! The most interesting modules here are `syntax_node` (which defines concrete
13//! syntax tree) and `ast` (which defines abstract syntax tree on top of the
14//! CST). The actual parser live in a separate `parser` crate, though the
15//! lexer lives in this crate.
16//!
17//! See `api_walkthrough` test in this file for a quick API tour!
18//!
19//! [RFC]: <https://github.com/rust-lang/rfcs/pull/2256>
20//! [Swift]: <https://github.com/apple/swift/blob/13d593df6f359d0cb2fc81cfaac273297c539455/lib/Syntax/README.md>
21
22mod parsing;
23mod ptr;
24mod syntax_error;
25mod syntax_node;
26#[cfg(test)]
27mod tests;
28mod token_text;
29mod validation;
30
31pub mod algo;
32pub mod ast;
33#[doc(hidden)]
34pub mod fuzz;
35pub mod hacks;
36pub mod syntax_editor;
37pub mod ted;
38pub mod utils;
39
40use std::{marker::PhantomData, ops::Range};
41
42use stdx::format_to;
43use triomphe::Arc;
44
45pub use crate::{
46    ast::{AstNode, AstToken},
47    ptr::{AstPtr, SyntaxNodePtr},
48    syntax_error::SyntaxError,
49    syntax_node::{
50        PreorderWithTokens, RustLanguage, SyntaxElement, SyntaxElementChildren, SyntaxNode,
51        SyntaxNodeChildren, SyntaxToken, SyntaxTreeBuilder,
52    },
53    token_text::TokenText,
54};
55pub use parser::{Edition, SyntaxKind, T};
56pub use rowan::{
57    Direction, GreenNode, NodeOrToken, SyntaxText, TextRange, TextSize, TokenAtOffset, WalkEvent,
58    api::Preorder,
59};
60pub use rustc_literal_escaper as unescape;
61pub use smol_str::{SmolStr, SmolStrBuilder, ToSmolStr, format_smolstr};
62
63/// `Parse` is the result of the parsing: a syntax tree and a collection of
64/// errors.
65///
66/// Note that we always produce a syntax tree, even for completely invalid
67/// files.
68#[derive(Debug, PartialEq, Eq)]
69pub struct Parse<T> {
70    green: Option<GreenNode>,
71    errors: Option<Arc<[SyntaxError]>>,
72    _ty: PhantomData<fn() -> T>,
73}
74
75impl<T> Clone for Parse<T> {
76    fn clone(&self) -> Parse<T> {
77        Parse { green: self.green.clone(), errors: self.errors.clone(), _ty: PhantomData }
78    }
79}
80
81impl<T> Parse<T> {
82    fn new(green: GreenNode, errors: Vec<SyntaxError>) -> Parse<T> {
83        Parse {
84            green: Some(green),
85            errors: if errors.is_empty() { None } else { Some(errors.into()) },
86            _ty: PhantomData,
87        }
88    }
89
90    pub fn syntax_node(&self) -> SyntaxNode {
91        SyntaxNode::new_root(self.green.as_ref().unwrap().clone())
92    }
93
94    pub fn errors(&self) -> Vec<SyntaxError> {
95        let mut errors = if let Some(e) = self.errors.as_deref() { e.to_vec() } else { vec![] };
96        validation::validate(&self.syntax_node(), &mut errors);
97        errors
98    }
99}
100
101impl<T: AstNode> Parse<T> {
102    /// Converts this parse result into a parse result for an untyped syntax tree.
103    pub fn to_syntax(mut self) -> Parse<SyntaxNode> {
104        let green = self.green.take();
105        let errors = self.errors.take();
106        Parse { green, errors, _ty: PhantomData }
107    }
108
109    /// Gets the parsed syntax tree as a typed ast node.
110    ///
111    /// # Panics
112    ///
113    /// Panics if the root node cannot be casted into the typed ast node
114    /// (e.g. if it's an `ERROR` node).
115    pub fn tree(&self) -> T {
116        T::cast(self.syntax_node()).unwrap()
117    }
118
119    /// Converts from `Parse<T>` to [`Result<T, Vec<SyntaxError>>`].
120    pub fn ok(self) -> Result<T, Vec<SyntaxError>> {
121        match self.errors() {
122            errors if !errors.is_empty() => Err(errors),
123            _ => Ok(self.tree()),
124        }
125    }
126}
127
128impl Parse<SyntaxNode> {
129    pub fn cast<N: AstNode>(mut self) -> Option<Parse<N>> {
130        if N::cast(self.syntax_node()).is_some() {
131            Some(Parse { green: self.green.take(), errors: self.errors.take(), _ty: PhantomData })
132        } else {
133            None
134        }
135    }
136}
137
138impl Parse<SourceFile> {
139    pub fn debug_dump(&self) -> String {
140        let mut buf = format!("{:#?}", self.tree().syntax());
141        for err in self.errors() {
142            format_to!(buf, "error {:?}: {}\n", err.range(), err);
143        }
144        buf
145    }
146
147    pub fn reparse(&self, delete: TextRange, insert: &str, edition: Edition) -> Parse<SourceFile> {
148        self.incremental_reparse(delete, insert, edition)
149            .unwrap_or_else(|| self.full_reparse(delete, insert, edition))
150    }
151
152    fn incremental_reparse(
153        &self,
154        delete: TextRange,
155        insert: &str,
156        edition: Edition,
157    ) -> Option<Parse<SourceFile>> {
158        // FIXME: validation errors are not handled here
159        parsing::incremental_reparse(
160            self.tree().syntax(),
161            delete,
162            insert,
163            self.errors.as_deref().unwrap_or_default().iter().cloned(),
164            edition,
165        )
166        .map(|(green_node, errors, _reparsed_range)| Parse {
167            green: Some(green_node),
168            errors: if errors.is_empty() { None } else { Some(errors.into()) },
169            _ty: PhantomData,
170        })
171    }
172
173    fn full_reparse(&self, delete: TextRange, insert: &str, edition: Edition) -> Parse<SourceFile> {
174        let mut text = self.tree().syntax().text().to_string();
175        text.replace_range(Range::<usize>::from(delete), insert);
176        SourceFile::parse(&text, edition)
177    }
178}
179
180impl ast::Expr {
181    /// Parses an `ast::Expr` from `text`.
182    ///
183    /// Note that if the parsed root node is not a valid expression, [`Parse::tree`] will panic.
184    /// For example:
185    /// ```rust,should_panic
186    /// # use syntax::{ast, Edition};
187    /// ast::Expr::parse("let fail = true;", Edition::CURRENT).tree();
188    /// ```
189    pub fn parse(text: &str, edition: Edition) -> Parse<ast::Expr> {
190        let _p = tracing::info_span!("Expr::parse").entered();
191        let (green, errors) = parsing::parse_text_at(text, parser::TopEntryPoint::Expr, edition);
192        let root = SyntaxNode::new_root(green.clone());
193
194        assert!(
195            ast::Expr::can_cast(root.kind()) || root.kind() == SyntaxKind::ERROR,
196            "{:?} isn't an expression",
197            root.kind()
198        );
199        Parse::new(green, errors)
200    }
201}
202
203#[cfg(not(no_salsa_async_drops))]
204impl<T> Drop for Parse<T> {
205    fn drop(&mut self) {
206        let Some(green) = self.green.take() else {
207            return;
208        };
209        static PARSE_DROP_THREAD: std::sync::OnceLock<std::sync::mpsc::Sender<GreenNode>> =
210            std::sync::OnceLock::new();
211        PARSE_DROP_THREAD
212            .get_or_init(|| {
213                let (sender, receiver) = std::sync::mpsc::channel::<GreenNode>();
214                std::thread::Builder::new()
215                    .name("ParseNodeDropper".to_owned())
216                    .spawn(move || receiver.iter().for_each(drop))
217                    .unwrap();
218                sender
219            })
220            .send(green)
221            .unwrap();
222    }
223}
224
225/// `SourceFile` represents a parse tree for a single Rust file.
226pub use crate::ast::SourceFile;
227
228impl SourceFile {
229    pub fn parse(text: &str, edition: Edition) -> Parse<SourceFile> {
230        let _p = tracing::info_span!("SourceFile::parse").entered();
231        let (green, errors) = parsing::parse_text(text, edition);
232        let root = SyntaxNode::new_root(green.clone());
233
234        assert_eq!(root.kind(), SyntaxKind::SOURCE_FILE);
235        Parse::new(green, errors)
236    }
237}
238
239/// Matches a `SyntaxNode` against an `ast` type.
240///
241/// # Example:
242///
243/// ```ignore
244/// match_ast! {
245///     match node {
246///         ast::CallExpr(it) => { ... },
247///         ast::MethodCallExpr(it) => { ... },
248///         ast::MacroCall(it) => { ... },
249///         _ => None,
250///     }
251/// }
252/// ```
253#[macro_export]
254macro_rules! match_ast {
255    (match $node:ident { $($tt:tt)* }) => { $crate::match_ast!(match ($node) { $($tt)* }) };
256
257    (match ($node:expr) {
258        $( $( $path:ident )::+ ($it:pat) => $res:expr, )*
259        _ => $catch_all:expr $(,)?
260    }) => {{
261        $( if let Some($it) = $($path::)+cast($node.clone()) { $res } else )*
262        { $catch_all }
263    }};
264}
265
266/// This test does not assert anything and instead just shows off the crate's
267/// API.
268#[test]
269fn api_walkthrough() {
270    use ast::{HasModuleItem, HasName};
271
272    let source_code = "
273        fn foo() {
274            1 + 1
275        }
276    ";
277    // `SourceFile` is the main entry point.
278    //
279    // The `parse` method returns a `Parse` -- a pair of syntax tree and a list
280    // of errors. That is, syntax tree is constructed even in presence of errors.
281    let parse = SourceFile::parse(source_code, parser::Edition::CURRENT);
282    assert!(parse.errors().is_empty());
283
284    // The `tree` method returns an owned syntax node of type `SourceFile`.
285    // Owned nodes are cheap: inside, they are `Rc` handles to the underling data.
286    let file: SourceFile = parse.tree();
287
288    // `SourceFile` is the root of the syntax tree. We can iterate file's items.
289    // Let's fetch the `foo` function.
290    let mut func = None;
291    for item in file.items() {
292        match item {
293            ast::Item::Fn(f) => func = Some(f),
294            _ => unreachable!(),
295        }
296    }
297    let func: ast::Fn = func.unwrap();
298
299    // Each AST node has a bunch of getters for children. All getters return
300    // `Option`s though, to account for incomplete code. Some getters are common
301    // for several kinds of node. In this case, a trait like `ast::NameOwner`
302    // usually exists. By convention, all ast types should be used with `ast::`
303    // qualifier.
304    let name: Option<ast::Name> = func.name();
305    let name = name.unwrap();
306    assert_eq!(name.text(), "foo");
307
308    // Let's get the `1 + 1` expression!
309    let body: ast::BlockExpr = func.body().unwrap();
310    let stmt_list: ast::StmtList = body.stmt_list().unwrap();
311    let expr: ast::Expr = stmt_list.tail_expr().unwrap();
312
313    // Enums are used to group related ast nodes together, and can be used for
314    // matching. However, because there are no public fields, it's possible to
315    // match only the top level enum: that is the price we pay for increased API
316    // flexibility
317    let bin_expr: &ast::BinExpr = match &expr {
318        ast::Expr::BinExpr(e) => e,
319        _ => unreachable!(),
320    };
321
322    // Besides the "typed" AST API, there's an untyped CST one as well.
323    // To switch from AST to CST, call `.syntax()` method:
324    let expr_syntax: &SyntaxNode = expr.syntax();
325
326    // Note how `expr` and `bin_expr` are in fact the same node underneath:
327    assert!(expr_syntax == bin_expr.syntax());
328
329    // To go from CST to AST, `AstNode::cast` function is used:
330    let _expr: ast::Expr = match ast::Expr::cast(expr_syntax.clone()) {
331        Some(e) => e,
332        None => unreachable!(),
333    };
334
335    // The two properties each syntax node has is a `SyntaxKind`:
336    assert_eq!(expr_syntax.kind(), SyntaxKind::BIN_EXPR);
337
338    // And text range:
339    assert_eq!(expr_syntax.text_range(), TextRange::new(32.into(), 37.into()));
340
341    // You can get node's text as a `SyntaxText` object, which will traverse the
342    // tree collecting token's text:
343    let text: SyntaxText = expr_syntax.text();
344    assert_eq!(text.to_string(), "1 + 1");
345
346    // There's a bunch of traversal methods on `SyntaxNode`:
347    assert_eq!(expr_syntax.parent().as_ref(), Some(stmt_list.syntax()));
348    assert_eq!(stmt_list.syntax().first_child_or_token().map(|it| it.kind()), Some(T!['{']));
349    assert_eq!(
350        expr_syntax.next_sibling_or_token().map(|it| it.kind()),
351        Some(SyntaxKind::WHITESPACE)
352    );
353
354    // As well as some iterator helpers:
355    let f = expr_syntax.ancestors().find_map(ast::Fn::cast);
356    assert_eq!(f, Some(func));
357    assert!(expr_syntax.siblings_with_tokens(Direction::Next).any(|it| it.kind() == T!['}']));
358    assert_eq!(
359        expr_syntax.descendants_with_tokens().count(),
360        8, // 5 tokens `1`, ` `, `+`, ` `, `1`
361           // 2 child literal expressions: `1`, `1`
362           // 1 the node itself: `1 + 1`
363    );
364
365    // There's also a `preorder` method with a more fine-grained iteration control:
366    let mut buf = String::new();
367    let mut indent = 0;
368    for event in expr_syntax.preorder_with_tokens() {
369        match event {
370            WalkEvent::Enter(node) => {
371                let text = match &node {
372                    NodeOrToken::Node(it) => it.text().to_string(),
373                    NodeOrToken::Token(it) => it.text().to_owned(),
374                };
375                format_to!(buf, "{:indent$}{:?} {:?}\n", " ", text, node.kind(), indent = indent);
376                indent += 2;
377            }
378            WalkEvent::Leave(_) => indent -= 2,
379        }
380    }
381    assert_eq!(indent, 0);
382    assert_eq!(
383        buf.trim(),
384        r#"
385"1 + 1" BIN_EXPR
386  "1" LITERAL
387    "1" INT_NUMBER
388  " " WHITESPACE
389  "+" PLUS
390  " " WHITESPACE
391  "1" LITERAL
392    "1" INT_NUMBER
393"#
394        .trim()
395    );
396
397    // To recursively process the tree, there are three approaches:
398    // 1. explicitly call getter methods on AST nodes.
399    // 2. use descendants and `AstNode::cast`.
400    // 3. use descendants and `match_ast!`.
401    //
402    // Here's how the first one looks like:
403    let exprs_cast: Vec<String> = file
404        .syntax()
405        .descendants()
406        .filter_map(ast::Expr::cast)
407        .map(|expr| expr.syntax().text().to_string())
408        .collect();
409
410    // An alternative is to use a macro.
411    let mut exprs_visit = Vec::new();
412    for node in file.syntax().descendants() {
413        match_ast! {
414            match node {
415                ast::Expr(it) => {
416                    let res = it.syntax().text().to_string();
417                    exprs_visit.push(res);
418                },
419                _ => (),
420            }
421        }
422    }
423    assert_eq!(exprs_cast, exprs_visit);
424}
ra_ap_syntax/lib.rs

ra_ap_syntax/
lib.rs
