oak_handlebars/lexer/
mod.rs

1#![doc = include_str!("readme.md")]
2use crate::language::HandlebarsLanguage;
3pub mod token_type;
4pub use token_type::HandlebarsTokenType;
5
6use crate::lexer::token_type::HandlebarsTokenType as T;
7use oak_core::{
8    Lexer, LexerCache, LexerState, OakError, Range,
9    lexer::{LexOutput, StringConfig, WhitespaceConfig},
10    source::Source,
11};
12use std::sync::LazyLock;
13
14type State<'a, S> = LexerState<'a, S, HandlebarsLanguage>;
15
16// Scanner configurations
17static HB_WHITESPACE: LazyLock<WhitespaceConfig> = LazyLock::new(|| WhitespaceConfig { unicode_whitespace: true });
18static HB_STRING_DOUBLE: LazyLock<StringConfig> = LazyLock::new(|| StringConfig { quotes: &['"'], escape: Some('\\') });
19static HB_STRING_SINGLE: LazyLock<StringConfig> = LazyLock::new(|| StringConfig { quotes: &['\''], escape: Some('\\') });
20
21#[derive(Clone)]
22pub struct HandlebarsLexer<'config> {
23    _config: &'config HandlebarsLanguage,
24}
25
26impl<'config> Lexer<HandlebarsLanguage> for HandlebarsLexer<'config> {
27    fn lex<'a, S: Source + ?Sized>(&self, source: &S, _edits: &[oak_core::TextEdit], cache: &'a mut impl LexerCache<HandlebarsLanguage>) -> LexOutput<HandlebarsLanguage> {
28        let mut state: State<'_, S> = LexerState::new(source);
29        let result = self.run(&mut state);
30        if result.is_ok() {
31            state.add_eof()
32        }
33        state.finish_with_cache(result, cache)
34    }
35}
36
37impl<'config> HandlebarsLexer<'config> {
38    pub fn new(config: &'config HandlebarsLanguage) -> Self {
39        Self { _config: config }
40    }
41
42    fn run<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> Result<(), OakError> {
43        while state.not_at_end() {
44            let safe_point = state.get_position();
45
46            if self.skip_whitespace(state) {
47                continue;
48            }
49
50            if self.skip_newline(state) {
51                continue;
52            }
53
54            if self.lex_comment(state) {
55                continue;
56            }
57
58            if self.lex_handlebars_expression(state) {
59                continue;
60            }
61
62            if self.lex_string_literal(state) {
63                continue;
64            }
65
66            if self.lex_number_literal(state) {
67                continue;
68            }
69
70            if self.lex_identifier(state) {
71                continue;
72            }
73
74            if self.lex_single_char_tokens(state) {
75                continue;
76            }
77
78            if self.lex_content(state) {
79                continue;
80            }
81
82            state.advance_if_dead_lock(safe_point)
83        }
84
85        Ok(())
86    }
87
88    fn skip_whitespace<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
89        HB_WHITESPACE.scan(state, HandlebarsTokenType::Whitespace)
90    }
91
92    fn skip_newline<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
93        if state.current() == Some('\n') || state.current() == Some('\r') {
94            let start = state.get_position();
95            state.advance(1);
96            if state.current() == Some('\n') && state.peek() == Some('\r') {
97                state.advance(1)
98            }
99            let end = state.get_position();
100            state.add_token(HandlebarsTokenType::Newline, start, end);
101            true
102        }
103        else {
104            false
105        }
106    }
107
108    fn lex_comment<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
109        if state.current() == Some('{') && state.peek() == Some('{') {
110            if state.peek_next_n(2) == Some('!') && state.peek_next_n(3) == Some('-') && state.peek_next_n(4) == Some('-') {
111                let start = state.get_position();
112                state.advance(5);
113                while state.not_at_end() {
114                    if state.current() == Some('-') && state.peek() == Some('-') && state.peek_next_n(2) == Some('}') && state.peek_next_n(3) == Some('}') {
115                        state.advance(4);
116                        let end = state.get_position();
117                        state.add_token(HandlebarsTokenType::Comment, start, end);
118                        return true;
119                    }
120                    state.advance(1);
121                }
122                return true;
123            }
124            else if state.peek_next_n(2) == Some('!') {
125                let start = state.get_position();
126                state.advance(3);
127                while state.not_at_end() {
128                    if state.current() == Some('}') && state.peek() == Some('}') {
129                        state.advance(2);
130                        let end = state.get_position();
131                        state.add_token(HandlebarsTokenType::Comment, start, end);
132                        return true;
133                    }
134                    state.advance(1);
135                }
136                return true;
137            }
138        }
139        false
140    }
141
142    fn lex_handlebars_expression<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
143        if state.current() == Some('{') && state.peek() == Some('{') {
144            let start = state.get_position();
145            if state.peek_next_n(2) == Some('{') {
146                state.advance(3);
147                state.add_token(HandlebarsTokenType::OpenUnescaped, start, state.get_position());
148            }
149            else {
150                state.advance(2);
151                state.add_token(HandlebarsTokenType::Open, start, state.get_position());
152            }
153            true
154        }
155        else if state.current() == Some('}') && state.peek() == Some('}') {
156            let start = state.get_position();
157            if state.peek_next_n(2) == Some('}') {
158                state.advance(3);
159                state.add_token(HandlebarsTokenType::CloseUnescaped, start, state.get_position());
160            }
161            else {
162                state.advance(2);
163                state.add_token(HandlebarsTokenType::Close, start, state.get_position());
164            }
165            true
166        }
167        else {
168            false
169        }
170    }
171
172    fn lex_string_literal<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
173        let config = if state.current() == Some('"') {
174            &*HB_STRING_DOUBLE
175        }
176        else if state.current() == Some('\'') {
177            &*HB_STRING_SINGLE
178        }
179        else {
180            return false;
181        };
182
183        config.scan(state, HandlebarsTokenType::StringLiteral)
184    }
185
186    fn lex_number_literal<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
187        if let Some(c) = state.current() {
188            if c.is_ascii_digit() {
189                let start = state.get_position();
190                while let Some(c) = state.current() {
191                    if c.is_ascii_digit() || c == '.' { state.advance(1) } else { break }
192                }
193                let end = state.get_position();
194                state.add_token(HandlebarsTokenType::NumberLiteral, start, end);
195                true
196            }
197            else {
198                false
199            }
200        }
201        else {
202            false
203        }
204    }
205
206    fn lex_identifier<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
207        if let Some(c) = state.current() {
208            if c.is_alphabetic() || c == '_' || c == '@' {
209                let start = state.get_position();
210                while let Some(c) = state.current() {
211                    if c.is_alphanumeric() || c == '_' || c == '-' || c == '.' { state.advance(1) } else { break }
212                }
213                let end = state.get_position();
214                let text = state.get_text_in(Range { start, end });
215                let kind = match text.as_ref() {
216                    "else" => HandlebarsTokenType::Else,
217                    "true" | "false" => HandlebarsTokenType::BooleanLiteral,
218                    _ => HandlebarsTokenType::Identifier,
219                };
220                state.add_token(kind, start, end);
221                true
222            }
223            else {
224                false
225            }
226        }
227        else {
228            false
229        }
230    }
231
232    fn lex_single_char_tokens<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
233        if let Some(c) = state.current() {
234            let start = state.get_position();
235            let kind = match c {
236                '(' => HandlebarsTokenType::LeftParen,
237                ')' => HandlebarsTokenType::RightParen,
238                '[' => HandlebarsTokenType::LeftBracket,
239                ']' => HandlebarsTokenType::RightBracket,
240                '=' => HandlebarsTokenType::Equal,
241                '|' => HandlebarsTokenType::Pipe,
242                '#' => HandlebarsTokenType::Hash,
243                '.' => HandlebarsTokenType::Dot,
244                '/' => HandlebarsTokenType::Slash,
245                '@' => HandlebarsTokenType::At,
246                '^' => HandlebarsTokenType::Caret,
247                _ => return false,
248            };
249            state.advance(1);
250            let end = state.get_position();
251            state.add_token(kind, start, end);
252            true
253        }
254        else {
255            false
256        }
257    }
258
259    fn lex_content<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
260        let start = state.get_position();
261        let mut count = 0;
262
263        while let Some(c) = state.current() {
264            if c == '{' && state.peek() == Some('{') {
265                break;
266            }
267            state.advance(1);
268            count += 1
269        }
270
271        if count > 0 {
272            let end = state.get_position();
273            state.add_token(HandlebarsTokenType::Content, start, end);
274            true
275        }
276        else {
277            false
278        }
279    }
280}
oak_handlebars/lexer/mod.rs

oak_handlebars/lexer/
mod.rs
