rsjsonnet_lang/parser/
mod.rs

1//! A parser for the Jsonnet language.
2//!
3//! # Example
4//!
5//! ```
6//! let source = b"local add_one(x) = x + 1; add_one(2)";
7//!
8//! let arena = rsjsonnet_lang::arena::Arena::new();
9//! let ast_arena = rsjsonnet_lang::arena::Arena::new();
10//! let str_interner = rsjsonnet_lang::interner::StrInterner::new();
11//! let mut span_mgr = rsjsonnet_lang::span::SpanManager::new();
12//! let (span_ctx, _) = span_mgr.insert_source_context(source.len());
13//!
14//! // First, lex the source excluding whitespace and comments.
15//! let lexer = rsjsonnet_lang::lexer::Lexer::new(
16//!     &arena,
17//!     &ast_arena,
18//!     &str_interner,
19//!     &mut span_mgr,
20//!     span_ctx,
21//!     source,
22//! );
23//! let tokens = lexer.lex_to_eof(false).unwrap();
24//!
25//! // Create the parser.
26//! let parser = rsjsonnet_lang::parser::Parser::new(
27//!     &arena,
28//!     &ast_arena,
29//!     &str_interner,
30//!     &mut span_mgr,
31//!     tokens,
32//! );
33//!
34//! // Parse the source.
35//! let ast_root = parser.parse_root_expr().unwrap();
36//! ```
37
38use crate::arena::Arena;
39use crate::ast;
40use crate::interner::StrInterner;
41use crate::span::{SpanId, SpanManager};
42use crate::token::{Number, STokenKind, Token, TokenKind};
43
44mod error;
45mod expr;
46
47pub use error::{ActualToken, ExpectedToken, ParseError};
48
49pub struct Parser<'a, 'p, 'ast> {
50    arena: &'p Arena,
51    ast_arena: &'ast Arena,
52    str_interner: &'a StrInterner<'p>,
53    span_mgr: &'a mut SpanManager,
54    curr_token: Token<'p, 'ast>,
55    rem_tokens: std::vec::IntoIter<Token<'p, 'ast>>,
56    expected_things: Vec<ExpectedToken>,
57}
58
59impl<'a, 'p, 'ast> Parser<'a, 'p, 'ast> {
60    /// Creates a new parser that operates on `tokens`.
61    ///
62    /// `arena` is used to allocate interned strings and `ast_arena` is used to
63    /// allocate most of AST nodes. This allows to free most of the AST while
64    /// keeping interned strings.
65    ///
66    /// `tokens` must not include [whitespace](TokenKind::Whitespace)
67    /// or [comments](TokenKind::Comment) and must end with an
68    /// [end-of-file](TokenKind::EndOfFile) token.
69    pub fn new(
70        arena: &'p Arena,
71        ast_arena: &'ast Arena,
72        str_interner: &'a StrInterner<'p>,
73        span_mgr: &'a mut SpanManager,
74        tokens: Vec<Token<'p, 'ast>>,
75    ) -> Self {
76        let mut token_iter = tokens.into_iter();
77        let first_token = token_iter.next().expect("passed an empty token slice");
78        Self {
79            arena,
80            ast_arena,
81            str_interner,
82            span_mgr,
83            curr_token: first_token,
84            rem_tokens: token_iter,
85            expected_things: Vec::new(),
86        }
87    }
88
89    fn next_token(&mut self) -> Token<'p, 'ast> {
90        let next_token = self.rem_tokens.next().unwrap();
91        let token = std::mem::replace(&mut self.curr_token, next_token);
92
93        self.expected_things.clear();
94
95        token
96    }
97
98    #[cold]
99    #[must_use]
100    fn report_expected(&mut self) -> ParseError {
101        ParseError::Expected {
102            span: self.curr_token.span,
103            expected: self.expected_things.drain(..).collect(),
104            instead: ActualToken::from_token_kind(&self.curr_token.kind),
105        }
106    }
107
108    #[inline]
109    fn eat_eof(&mut self, add_to_expected: bool) -> bool {
110        if matches!(self.curr_token.kind, TokenKind::EndOfFile) {
111            assert!(self.rem_tokens.as_slice().is_empty());
112            self.expected_things.clear();
113            true
114        } else {
115            if add_to_expected {
116                self.expected_things.push(ExpectedToken::EndOfFile);
117            }
118            false
119        }
120    }
121
122    #[inline]
123    fn eat_simple(&mut self, kind: STokenKind, add_to_expected: bool) -> Option<SpanId> {
124        if matches!(self.curr_token.kind, TokenKind::Simple(k) if k == kind) {
125            let span = self.curr_token.span;
126            self.next_token();
127            Some(span)
128        } else {
129            if add_to_expected {
130                self.expected_things.push(ExpectedToken::Simple(kind));
131            }
132            None
133        }
134    }
135
136    #[inline]
137    fn expect_simple(
138        &mut self,
139        kind: STokenKind,
140        add_to_expected: bool,
141    ) -> Result<SpanId, ParseError> {
142        if let Some(span) = self.eat_simple(kind, add_to_expected) {
143            Ok(span)
144        } else {
145            Err(self.report_expected())
146        }
147    }
148
149    #[inline]
150    fn eat_ident(&mut self, add_to_expected: bool) -> Option<ast::Ident<'p>> {
151        if let TokenKind::Ident(value) = self.curr_token.kind {
152            let span = self.curr_token.span;
153            self.next_token();
154            Some(ast::Ident { value, span })
155        } else {
156            if add_to_expected {
157                self.expected_things.push(ExpectedToken::Ident);
158            }
159            None
160        }
161    }
162
163    #[inline]
164    fn expect_ident(&mut self, add_to_expected: bool) -> Result<ast::Ident<'p>, ParseError> {
165        if let Some(ident) = self.eat_ident(add_to_expected) {
166            Ok(ident)
167        } else {
168            Err(self.report_expected())
169        }
170    }
171
172    #[inline]
173    fn eat_number(&mut self, add_to_expected: bool) -> Option<(Number<'ast>, SpanId)> {
174        if let TokenKind::Number(n) = self.curr_token.kind {
175            let span = self.curr_token.span;
176            self.next_token();
177            Some((n, span))
178        } else {
179            if add_to_expected {
180                self.expected_things.push(ExpectedToken::Number);
181            }
182            None
183        }
184    }
185
186    #[inline]
187    fn eat_string(&mut self, add_to_expected: bool) -> Option<(&'ast str, SpanId)> {
188        if let TokenKind::String(s) = self.curr_token.kind {
189            let span = self.curr_token.span;
190            self.next_token();
191            Some((s, span))
192        } else {
193            if add_to_expected {
194                self.expected_things.push(ExpectedToken::String);
195            }
196            None
197        }
198    }
199
200    #[inline]
201    fn eat_text_block(&mut self, add_to_expected: bool) -> Option<(&'ast str, SpanId)> {
202        if let TokenKind::TextBlock(s) = self.curr_token.kind {
203            let span = self.curr_token.span;
204            self.next_token();
205            Some((s, span))
206        } else {
207            if add_to_expected {
208                self.expected_things.push(ExpectedToken::TextBlock);
209            }
210            None
211        }
212    }
213
214    #[inline]
215    fn eat_visibility(&mut self, add_to_expected: bool) -> Option<ast::Visibility> {
216        if self
217            .eat_simple(STokenKind::Colon, add_to_expected)
218            .is_some()
219        {
220            Some(ast::Visibility::Default)
221        } else if self
222            .eat_simple(STokenKind::ColonColon, add_to_expected)
223            .is_some()
224        {
225            Some(ast::Visibility::Hidden)
226        } else if self
227            .eat_simple(STokenKind::ColonColonColon, add_to_expected)
228            .is_some()
229        {
230            Some(ast::Visibility::ForceVisible)
231        } else {
232            None
233        }
234    }
235
236    #[inline]
237    fn eat_plus_visibility(&mut self, add_to_expected: bool) -> Option<(bool, ast::Visibility)> {
238        if self
239            .eat_simple(STokenKind::Colon, add_to_expected)
240            .is_some()
241        {
242            Some((false, ast::Visibility::Default))
243        } else if self
244            .eat_simple(STokenKind::ColonColon, add_to_expected)
245            .is_some()
246        {
247            Some((false, ast::Visibility::Hidden))
248        } else if self
249            .eat_simple(STokenKind::ColonColonColon, add_to_expected)
250            .is_some()
251        {
252            Some((false, ast::Visibility::ForceVisible))
253        } else if self
254            .eat_simple(STokenKind::PlusColon, add_to_expected)
255            .is_some()
256        {
257            Some((true, ast::Visibility::Default))
258        } else if self
259            .eat_simple(STokenKind::PlusColonColon, add_to_expected)
260            .is_some()
261        {
262            Some((true, ast::Visibility::Hidden))
263        } else if self
264            .eat_simple(STokenKind::PlusColonColonColon, add_to_expected)
265            .is_some()
266        {
267            Some((true, ast::Visibility::ForceVisible))
268        } else {
269            None
270        }
271    }
272
273    #[inline]
274    fn peek_simple(&self, skind: STokenKind, i: usize) -> bool {
275        if i == 0 {
276            matches!(self.curr_token.kind, TokenKind::Simple(k) if k == skind)
277        } else if let Some(Token { kind, .. }) = self.rem_tokens.as_slice().get(i - 1) {
278            matches!(*kind, TokenKind::Simple(k) if k == skind)
279        } else {
280            false
281        }
282    }
283
284    #[inline]
285    fn peek_ident(&self, i: usize) -> bool {
286        if i == 0 {
287            matches!(self.curr_token.kind, TokenKind::Ident(_))
288        } else {
289            matches!(
290                self.rem_tokens.as_slice().get(i - 1),
291                Some(Token {
292                    kind: TokenKind::Ident(_),
293                    ..
294                })
295            )
296        }
297    }
298
299    /// Parses the tokens into an expression.
300    pub fn parse_root_expr(mut self) -> Result<ast::Expr<'p, 'ast>, ParseError> {
301        let expr = self.parse_expr()?;
302        if self.eat_eof(true) {
303            Ok(expr)
304        } else {
305            Err(self.report_expected())
306        }
307    }
308}
rsjsonnet_lang/parser/mod.rs

rsjsonnet_lang/parser/
mod.rs
