ra_ap_syntax/
lib.rs

1//! Syntax Tree library used throughout the rust-analyzer.
2//!
3//! Properties:
4//!   - easy and fast incremental re-parsing
5//!   - graceful handling of errors
6//!   - full-fidelity representation (*any* text can be precisely represented as
7//!     a syntax tree)
8//!
9//! For more information, see the [RFC]. Current implementation is inspired by
10//! the [Swift] one.
11//!
12//! The most interesting modules here are `syntax_node` (which defines concrete
13//! syntax tree) and `ast` (which defines abstract syntax tree on top of the
14//! CST). The actual parser live in a separate `parser` crate, though the
15//! lexer lives in this crate.
16//!
17//! See `api_walkthrough` test in this file for a quick API tour!
18//!
19//! [RFC]: <https://github.com/rust-lang/rfcs/pull/2256>
20//! [Swift]: <https://github.com/apple/swift/blob/13d593df6f359d0cb2fc81cfaac273297c539455/lib/Syntax/README.md>
21
22mod parsing;
23mod ptr;
24mod syntax_error;
25mod syntax_node;
26#[cfg(test)]
27mod tests;
28mod token_text;
29mod validation;
30
31pub mod algo;
32pub mod ast;
33#[doc(hidden)]
34pub mod fuzz;
35pub mod hacks;
36pub mod syntax_editor;
37pub mod ted;
38pub mod utils;
39
40use std::{marker::PhantomData, ops::Range};
41
42use stdx::format_to;
43use triomphe::Arc;
44
45pub use crate::{
46    ast::{AstNode, AstToken},
47    ptr::{AstPtr, SyntaxNodePtr},
48    syntax_error::SyntaxError,
49    syntax_node::{
50        PreorderWithTokens, RustLanguage, SyntaxElement, SyntaxElementChildren, SyntaxNode,
51        SyntaxNodeChildren, SyntaxToken, SyntaxTreeBuilder,
52    },
53    token_text::TokenText,
54};
55pub use parser::{Edition, SyntaxKind, T};
56pub use rowan::{
57    Direction, GreenNode, NodeOrToken, SyntaxText, TextRange, TextSize, TokenAtOffset, WalkEvent,
58    api::Preorder,
59};
60pub use rustc_literal_escaper as unescape;
61pub use smol_str::{SmolStr, SmolStrBuilder, ToSmolStr, format_smolstr};
62
63/// `Parse` is the result of the parsing: a syntax tree and a collection of
64/// errors.
65///
66/// Note that we always produce a syntax tree, even for completely invalid
67/// files.
68#[derive(Debug, PartialEq, Eq)]
69pub struct Parse<T> {
70    green: GreenNode,
71    errors: Option<Arc<[SyntaxError]>>,
72    _ty: PhantomData<fn() -> T>,
73}
74
75impl<T> Clone for Parse<T> {
76    fn clone(&self) -> Parse<T> {
77        Parse { green: self.green.clone(), errors: self.errors.clone(), _ty: PhantomData }
78    }
79}
80
81impl<T> Parse<T> {
82    fn new(green: GreenNode, errors: Vec<SyntaxError>) -> Parse<T> {
83        Parse {
84            green,
85            errors: if errors.is_empty() { None } else { Some(errors.into()) },
86            _ty: PhantomData,
87        }
88    }
89
90    pub fn syntax_node(&self) -> SyntaxNode {
91        SyntaxNode::new_root(self.green.clone())
92    }
93
94    pub fn errors(&self) -> Vec<SyntaxError> {
95        let mut errors = if let Some(e) = self.errors.as_deref() { e.to_vec() } else { vec![] };
96        validation::validate(&self.syntax_node(), &mut errors);
97        errors
98    }
99}
100
101impl<T: AstNode> Parse<T> {
102    /// Converts this parse result into a parse result for an untyped syntax tree.
103    pub fn to_syntax(self) -> Parse<SyntaxNode> {
104        Parse { green: self.green, errors: self.errors, _ty: PhantomData }
105    }
106
107    /// Gets the parsed syntax tree as a typed ast node.
108    ///
109    /// # Panics
110    ///
111    /// Panics if the root node cannot be casted into the typed ast node
112    /// (e.g. if it's an `ERROR` node).
113    pub fn tree(&self) -> T {
114        T::cast(self.syntax_node()).unwrap()
115    }
116
117    /// Converts from `Parse<T>` to [`Result<T, Vec<SyntaxError>>`].
118    pub fn ok(self) -> Result<T, Vec<SyntaxError>> {
119        match self.errors() {
120            errors if !errors.is_empty() => Err(errors),
121            _ => Ok(self.tree()),
122        }
123    }
124}
125
126impl Parse<SyntaxNode> {
127    pub fn cast<N: AstNode>(self) -> Option<Parse<N>> {
128        if N::cast(self.syntax_node()).is_some() {
129            Some(Parse { green: self.green, errors: self.errors, _ty: PhantomData })
130        } else {
131            None
132        }
133    }
134}
135
136impl Parse<SourceFile> {
137    pub fn debug_dump(&self) -> String {
138        let mut buf = format!("{:#?}", self.tree().syntax());
139        for err in self.errors() {
140            format_to!(buf, "error {:?}: {}\n", err.range(), err);
141        }
142        buf
143    }
144
145    pub fn reparse(&self, delete: TextRange, insert: &str, edition: Edition) -> Parse<SourceFile> {
146        self.incremental_reparse(delete, insert, edition)
147            .unwrap_or_else(|| self.full_reparse(delete, insert, edition))
148    }
149
150    fn incremental_reparse(
151        &self,
152        delete: TextRange,
153        insert: &str,
154        edition: Edition,
155    ) -> Option<Parse<SourceFile>> {
156        // FIXME: validation errors are not handled here
157        parsing::incremental_reparse(
158            self.tree().syntax(),
159            delete,
160            insert,
161            self.errors.as_deref().unwrap_or_default().iter().cloned(),
162            edition,
163        )
164        .map(|(green_node, errors, _reparsed_range)| Parse {
165            green: green_node,
166            errors: if errors.is_empty() { None } else { Some(errors.into()) },
167            _ty: PhantomData,
168        })
169    }
170
171    fn full_reparse(&self, delete: TextRange, insert: &str, edition: Edition) -> Parse<SourceFile> {
172        let mut text = self.tree().syntax().text().to_string();
173        text.replace_range(Range::<usize>::from(delete), insert);
174        SourceFile::parse(&text, edition)
175    }
176}
177
178impl ast::Expr {
179    /// Parses an `ast::Expr` from `text`.
180    ///
181    /// Note that if the parsed root node is not a valid expression, [`Parse::tree`] will panic.
182    /// For example:
183    /// ```rust,should_panic
184    /// # use syntax::{ast, Edition};
185    /// ast::Expr::parse("let fail = true;", Edition::CURRENT).tree();
186    /// ```
187    pub fn parse(text: &str, edition: Edition) -> Parse<ast::Expr> {
188        let _p = tracing::info_span!("Expr::parse").entered();
189        let (green, errors) = parsing::parse_text_at(text, parser::TopEntryPoint::Expr, edition);
190        let root = SyntaxNode::new_root(green.clone());
191
192        assert!(
193            ast::Expr::can_cast(root.kind()) || root.kind() == SyntaxKind::ERROR,
194            "{:?} isn't an expression",
195            root.kind()
196        );
197        Parse::new(green, errors)
198    }
199}
200
201/// `SourceFile` represents a parse tree for a single Rust file.
202pub use crate::ast::SourceFile;
203
204impl SourceFile {
205    pub fn parse(text: &str, edition: Edition) -> Parse<SourceFile> {
206        let _p = tracing::info_span!("SourceFile::parse").entered();
207        let (green, errors) = parsing::parse_text(text, edition);
208        let root = SyntaxNode::new_root(green.clone());
209
210        assert_eq!(root.kind(), SyntaxKind::SOURCE_FILE);
211        Parse::new(green, errors)
212    }
213}
214
215/// Matches a `SyntaxNode` against an `ast` type.
216///
217/// # Example:
218///
219/// ```ignore
220/// match_ast! {
221///     match node {
222///         ast::CallExpr(it) => { ... },
223///         ast::MethodCallExpr(it) => { ... },
224///         ast::MacroCall(it) => { ... },
225///         _ => None,
226///     }
227/// }
228/// ```
229#[macro_export]
230macro_rules! match_ast {
231    (match $node:ident { $($tt:tt)* }) => { $crate::match_ast!(match ($node) { $($tt)* }) };
232
233    (match ($node:expr) {
234        $( $( $path:ident )::+ ($it:pat) => $res:expr, )*
235        _ => $catch_all:expr $(,)?
236    }) => {{
237        $( if let Some($it) = $($path::)+cast($node.clone()) { $res } else )*
238        { $catch_all }
239    }};
240}
241
242/// This test does not assert anything and instead just shows off the crate's
243/// API.
244#[test]
245fn api_walkthrough() {
246    use ast::{HasModuleItem, HasName};
247
248    let source_code = "
249        fn foo() {
250            1 + 1
251        }
252    ";
253    // `SourceFile` is the main entry point.
254    //
255    // The `parse` method returns a `Parse` -- a pair of syntax tree and a list
256    // of errors. That is, syntax tree is constructed even in presence of errors.
257    let parse = SourceFile::parse(source_code, parser::Edition::CURRENT);
258    assert!(parse.errors().is_empty());
259
260    // The `tree` method returns an owned syntax node of type `SourceFile`.
261    // Owned nodes are cheap: inside, they are `Rc` handles to the underling data.
262    let file: SourceFile = parse.tree();
263
264    // `SourceFile` is the root of the syntax tree. We can iterate file's items.
265    // Let's fetch the `foo` function.
266    let mut func = None;
267    for item in file.items() {
268        match item {
269            ast::Item::Fn(f) => func = Some(f),
270            _ => unreachable!(),
271        }
272    }
273    let func: ast::Fn = func.unwrap();
274
275    // Each AST node has a bunch of getters for children. All getters return
276    // `Option`s though, to account for incomplete code. Some getters are common
277    // for several kinds of node. In this case, a trait like `ast::NameOwner`
278    // usually exists. By convention, all ast types should be used with `ast::`
279    // qualifier.
280    let name: Option<ast::Name> = func.name();
281    let name = name.unwrap();
282    assert_eq!(name.text(), "foo");
283
284    // Let's get the `1 + 1` expression!
285    let body: ast::BlockExpr = func.body().unwrap();
286    let stmt_list: ast::StmtList = body.stmt_list().unwrap();
287    let expr: ast::Expr = stmt_list.tail_expr().unwrap();
288
289    // Enums are used to group related ast nodes together, and can be used for
290    // matching. However, because there are no public fields, it's possible to
291    // match only the top level enum: that is the price we pay for increased API
292    // flexibility
293    let bin_expr: &ast::BinExpr = match &expr {
294        ast::Expr::BinExpr(e) => e,
295        _ => unreachable!(),
296    };
297
298    // Besides the "typed" AST API, there's an untyped CST one as well.
299    // To switch from AST to CST, call `.syntax()` method:
300    let expr_syntax: &SyntaxNode = expr.syntax();
301
302    // Note how `expr` and `bin_expr` are in fact the same node underneath:
303    assert!(expr_syntax == bin_expr.syntax());
304
305    // To go from CST to AST, `AstNode::cast` function is used:
306    let _expr: ast::Expr = match ast::Expr::cast(expr_syntax.clone()) {
307        Some(e) => e,
308        None => unreachable!(),
309    };
310
311    // The two properties each syntax node has is a `SyntaxKind`:
312    assert_eq!(expr_syntax.kind(), SyntaxKind::BIN_EXPR);
313
314    // And text range:
315    assert_eq!(expr_syntax.text_range(), TextRange::new(32.into(), 37.into()));
316
317    // You can get node's text as a `SyntaxText` object, which will traverse the
318    // tree collecting token's text:
319    let text: SyntaxText = expr_syntax.text();
320    assert_eq!(text.to_string(), "1 + 1");
321
322    // There's a bunch of traversal methods on `SyntaxNode`:
323    assert_eq!(expr_syntax.parent().as_ref(), Some(stmt_list.syntax()));
324    assert_eq!(stmt_list.syntax().first_child_or_token().map(|it| it.kind()), Some(T!['{']));
325    assert_eq!(
326        expr_syntax.next_sibling_or_token().map(|it| it.kind()),
327        Some(SyntaxKind::WHITESPACE)
328    );
329
330    // As well as some iterator helpers:
331    let f = expr_syntax.ancestors().find_map(ast::Fn::cast);
332    assert_eq!(f, Some(func));
333    assert!(expr_syntax.siblings_with_tokens(Direction::Next).any(|it| it.kind() == T!['}']));
334    assert_eq!(
335        expr_syntax.descendants_with_tokens().count(),
336        8, // 5 tokens `1`, ` `, `+`, ` `, `1`
337           // 2 child literal expressions: `1`, `1`
338           // 1 the node itself: `1 + 1`
339    );
340
341    // There's also a `preorder` method with a more fine-grained iteration control:
342    let mut buf = String::new();
343    let mut indent = 0;
344    for event in expr_syntax.preorder_with_tokens() {
345        match event {
346            WalkEvent::Enter(node) => {
347                let text = match &node {
348                    NodeOrToken::Node(it) => it.text().to_string(),
349                    NodeOrToken::Token(it) => it.text().to_owned(),
350                };
351                format_to!(buf, "{:indent$}{:?} {:?}\n", " ", text, node.kind(), indent = indent);
352                indent += 2;
353            }
354            WalkEvent::Leave(_) => indent -= 2,
355        }
356    }
357    assert_eq!(indent, 0);
358    assert_eq!(
359        buf.trim(),
360        r#"
361"1 + 1" BIN_EXPR
362  "1" LITERAL
363    "1" INT_NUMBER
364  " " WHITESPACE
365  "+" PLUS
366  " " WHITESPACE
367  "1" LITERAL
368    "1" INT_NUMBER
369"#
370        .trim()
371    );
372
373    // To recursively process the tree, there are three approaches:
374    // 1. explicitly call getter methods on AST nodes.
375    // 2. use descendants and `AstNode::cast`.
376    // 3. use descendants and `match_ast!`.
377    //
378    // Here's how the first one looks like:
379    let exprs_cast: Vec<String> = file
380        .syntax()
381        .descendants()
382        .filter_map(ast::Expr::cast)
383        .map(|expr| expr.syntax().text().to_string())
384        .collect();
385
386    // An alternative is to use a macro.
387    let mut exprs_visit = Vec::new();
388    for node in file.syntax().descendants() {
389        match_ast! {
390            match node {
391                ast::Expr(it) => {
392                    let res = it.syntax().text().to_string();
393                    exprs_visit.push(res);
394                },
395                _ => (),
396            }
397        }
398    }
399    assert_eq!(exprs_cast, exprs_visit);
400}
ra_ap_syntax/lib.rs

ra_ap_syntax/
lib.rs
