ludtwig_parser/parser/
source.rs

1use rowan::TextRange;
2
3use crate::lexer::Token;
4use crate::syntax::untyped::SyntaxKind;
5
6/// Wrapper around lexing tokens to only get the non-whitespace tokens back
7#[derive(Debug, Clone, Eq, PartialEq)]
8pub(super) struct Source<'source> {
9    tokens: &'source [Token<'source>],
10    cursor: usize,
11}
12
13impl<'source> Source<'source> {
14    pub(super) fn new(tokens: &'source [Token<'source>]) -> Self {
15        Self { tokens, cursor: 0 }
16    }
17
18    pub(super) fn next_token(&mut self) -> Option<&'source Token<'source>> {
19        self.eat_trivia();
20
21        let token = self.tokens.get(self.cursor)?;
22        self.cursor += 1;
23
24        Some(token)
25    }
26
27    pub(super) fn next_n_tokens(&mut self, n: usize) -> Vec<&'source Token<'source>> {
28        self.eat_trivia();
29
30        let token = self.tokens[self.cursor..].iter().take(n).collect();
31        self.cursor += n;
32
33        token
34    }
35
36    pub(super) fn peek_kind(&mut self) -> Option<SyntaxKind> {
37        self.eat_trivia();
38        self.peek_kind_raw()
39    }
40
41    pub(super) fn peek_token(&mut self) -> Option<&Token> {
42        self.eat_trivia();
43        self.peek_token_raw()
44    }
45
46    /// Lookahead is expensive!
47    /// This lookahead doesn't skip further trivia tokens and is only there for combining the next n lexer tokens!
48    /// for n of zero use `peek_token` instead!
49    pub(super) fn peek_nth_token(&mut self, n: usize) -> Option<&Token> {
50        self.eat_trivia();
51        self.tokens[self.cursor..].get(n)
52    }
53
54    pub(super) fn at_following(&mut self, set: &[SyntaxKind]) -> bool {
55        self.eat_trivia();
56        if self.cursor == self.tokens.len() {
57            return false; // end already reached
58        }
59
60        let mut tokens_iter = self.tokens[self.cursor..]
61            .iter()
62            .map(|t| t.kind)
63            .filter(|k| !k.is_trivia());
64        let mut set_iter = set.iter();
65
66        loop {
67            match (tokens_iter.next(), set_iter.next()) {
68                (Some(token), Some(set)) if token == *set => continue,
69                (None | Some(_), None) => return true,
70                _ => return false,
71            }
72        }
73    }
74
75    pub(super) fn at_following_content(&mut self, set: &[(SyntaxKind, Option<&str>)]) -> bool {
76        self.eat_trivia();
77        if self.cursor == self.tokens.len() {
78            return false; // end already reached
79        }
80
81        let mut tokens_iter = self.tokens[self.cursor..]
82            .iter()
83            .filter(|t| !t.kind.is_trivia());
84        let mut set_iter = set.iter();
85
86        loop {
87            match (tokens_iter.next(), set_iter.next()) {
88                (Some(token), Some((set_kind, set_content)))
89                    if token.kind == *set_kind
90                        && set_content.map_or(true, |content| content == token.text) =>
91                {
92                    continue
93                }
94                (None | Some(_), None) => return true,
95                _ => return false,
96            }
97        }
98    }
99
100    pub(super) fn last_token_range(&self) -> Option<TextRange> {
101        self.tokens.last().map(|Token { range, .. }| *range)
102    }
103
104    pub(super) fn get_pos(&self) -> usize {
105        self.cursor
106    }
107
108    fn eat_trivia(&mut self) {
109        while self.at_trivia() {
110            self.cursor += 1;
111        }
112    }
113
114    fn at_trivia(&self) -> bool {
115        self.peek_kind_raw().is_some_and(SyntaxKind::is_trivia)
116    }
117
118    fn peek_kind_raw(&self) -> Option<SyntaxKind> {
119        self.peek_token_raw().map(|Token { kind, .. }| *kind)
120    }
121
122    fn peek_token_raw(&self) -> Option<&Token> {
123        self.tokens.get(self.cursor)
124    }
125}
126
127#[cfg(test)]
128mod tests {
129    use crate::T;
130
131    use super::*;
132
133    #[test]
134    fn source_skip_whitespace() {
135        let tokens = vec![
136            Token::new_wrong_range(T![ws], "  "),
137            Token::new_wrong_range(T![lb], "\n"),
138            Token::new_wrong_range(T![word], "word"),
139            Token::new_wrong_range(T![lb], "\n"),
140            Token::new_wrong_range(T![ws], "  "),
141        ];
142
143        let mut source = Source::new(&tokens);
144        assert_eq!(source.peek_kind(), Some(T![word]));
145        assert_eq!(
146            source.next_token(),
147            Some(&Token::new_wrong_range(T![word], "word"))
148        );
149        assert_eq!(source.peek_kind(), None);
150        assert_eq!(source.next_token(), None);
151    }
152
153    #[test]
154    fn source_at_following() {
155        let tokens = vec![
156            Token::new_wrong_range(T![ws], "  "),
157            Token::new_wrong_range(T![lb], "\n"),
158            Token::new_wrong_range(T![word], "hello"),
159            Token::new_wrong_range(T![lb], "\n"),
160            Token::new_wrong_range(T![ws], "  "),
161            Token::new_wrong_range(T!["<"], "<"),
162            Token::new_wrong_range(T![lb], "\n"),
163            Token::new_wrong_range(T![">"], ">"),
164            Token::new_wrong_range(T![ws], "  "),
165        ];
166
167        let mut source = Source::new(&tokens);
168        assert!(source.at_following(&[T![word], T!["<"], T![">"]]));
169        assert!(source.at_following(&[T![word], T!["<"]]));
170        assert!(source.at_following(&[T![word]]));
171
172        assert!(!source.at_following(&[T![word], T!["<"], T![">"], T![word]]));
173        assert!(!source.at_following(&[T!["<"]]));
174        assert!(!source.at_following(&[T![word], T![">"]]));
175
176        source.next_token();
177        source.next_token();
178        source.next_token();
179
180        // nothing more to compare
181        assert!(!source.at_following(&[T![word]]));
182    }
183
184    #[test]
185    fn source_at_following_content() {
186        let tokens = vec![
187            Token::new_wrong_range(T![ws], "  "),
188            Token::new_wrong_range(T![lb], "\n"),
189            Token::new_wrong_range(T![word], "hello"),
190            Token::new_wrong_range(T![lb], "\n"),
191            Token::new_wrong_range(T![ws], "  "),
192            Token::new_wrong_range(T!["<"], "<"),
193            Token::new_wrong_range(T![lb], "\n"),
194            Token::new_wrong_range(T![">"], ">"),
195            Token::new_wrong_range(T![ws], "  "),
196        ];
197
198        let mut source = Source::new(&tokens);
199        assert!(source.at_following_content(&[
200            (T![word], Some("hello")),
201            (T!["<"], None),
202            (T![">"], None)
203        ]));
204        assert!(source.at_following_content(&[(T![word], None), (T!["<"], None), (T![">"], None)]));
205        assert!(source.at_following_content(&[(T![word], Some("hello")), (T!["<"], None)]));
206        assert!(source.at_following_content(&[(T![word], Some("hello"))]));
207        assert!(source.at_following_content(&[
208            (T![word], None),
209            (T!["<"], None),
210            (T![">"], Some(">"))
211        ]));
212
213        assert!(!source.at_following_content(&[(T![word], Some("nonExistent"))]));
214        assert!(!source.at_following_content(&[
215            (T![word], Some("nonExistent")),
216            (T!["<"], None),
217            (T![">"], None)
218        ]));
219
220        source.next_token();
221        source.next_token();
222        source.next_token();
223
224        // nothing more to compare
225        assert!(!source.at_following_content(&[(T![word], None)]));
226    }
227}
ludtwig_parser/parser/source.rs

ludtwig_parser/parser/
source.rs
