oak_tailwind/lexer/
mod.rs

1#![doc = include_str!("readme.md")]
2pub mod token_type;
3
4use crate::{language::TailwindLanguage, lexer::token_type::TailwindTokenType};
5use oak_core::{Lexer, LexerCache, LexerState, OakError, lexer::LexOutput, source::Source};
6
7/// Lexer for the Tailwind language.
8#[derive(Clone, Debug)]
9pub struct TailwindLexer<'config> {
10    /// Language configuration
11    _config: &'config TailwindLanguage,
12}
13
14type State<'a, S> = LexerState<'a, S, TailwindLanguage>;
15
16impl<'config> TailwindLexer<'config> {
17    /// Creates a new `TailwindLexer` with the given configuration.
18    pub fn new(config: &'config TailwindLanguage) -> Self {
19        Self { _config: config }
20    }
21}
22
23impl<'config> Lexer<TailwindLanguage> for TailwindLexer<'config> {
24    /// Tokenizes the source text into a sequence of Tailwind tokens.
25    fn lex<'a, S: Source + ?Sized>(&self, source: &S, _edits: &[oak_core::TextEdit], cache: &'a mut impl LexerCache<TailwindLanguage>) -> LexOutput<TailwindLanguage> {
26        let mut state = LexerState::new(source);
27        let result = self.run(&mut state);
28        if result.is_ok() {
29            state.add_eof()
30        }
31        state.finish_with_cache(result, cache)
32    }
33}
34
35impl<'config> TailwindLexer<'config> {
36    fn run<S: Source + ?Sized>(&self, state: &mut State<'_, S>) -> Result<(), OakError> {
37        while state.not_at_end() {
38            let safe_point = state.get_position();
39
40            if self.skip_whitespace(state) {
41                continue;
42            }
43
44            if self.skip_comment(state) {
45                continue;
46            }
47
48            if self.lex_string(state) {
49                continue;
50            }
51
52            if self.lex_number(state) {
53                continue;
54            }
55
56            if self.lex_punctuation(state) {
57                continue;
58            }
59
60            if self.lex_identifier(state) {
61                continue;
62            }
63
64            state.advance_if_dead_lock(safe_point)
65        }
66
67        Ok(())
68    }
69
70    fn skip_whitespace<S: Source + ?Sized>(&self, state: &mut State<'_, S>) -> bool {
71        let start = state.get_position();
72        let mut found = false;
73
74        while let Some(ch) = state.peek() {
75            if ch.is_whitespace() {
76                state.advance(ch.len_utf8());
77                found = true
78            }
79            else {
80                break;
81            }
82        }
83
84        if found {
85            state.add_token(TailwindTokenType::Whitespace, start, state.get_position())
86        }
87
88        found
89    }
90
91    fn skip_comment<S: Source + ?Sized>(&self, state: &mut State<'_, S>) -> bool {
92        let start = state.get_position();
93        if state.consume_if_starts_with("{#") {
94            while state.not_at_end() {
95                if state.consume_if_starts_with("#}") {
96                    break;
97                }
98                if let Some(ch) = state.peek() {
99                    state.advance(ch.len_utf8())
100                }
101            }
102            state.add_token(TailwindTokenType::Comment, start, state.get_position());
103            return true;
104        }
105        false
106    }
107
108    fn lex_string<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
109        let start = state.get_position();
110
111        if let Some(quote) = state.peek() {
112            if quote == '"' || quote == '\'' {
113                state.advance(1);
114
115                while let Some(ch) = state.peek() {
116                    if ch == quote {
117                        state.advance(1);
118                        break;
119                    }
120                    else if ch == '\\' {
121                        state.advance(1);
122                        if let Some(_) = state.peek() {
123                            state.advance(1)
124                        }
125                    }
126                    else {
127                        state.advance(ch.len_utf8())
128                    }
129                }
130
131                state.add_token(TailwindTokenType::String, start, state.get_position());
132                return true;
133            }
134        }
135
136        false
137    }
138
139    fn lex_number<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
140        let start = state.get_position();
141
142        if let Some(ch) = state.peek() {
143            if ch.is_ascii_digit() {
144                state.advance(1);
145
146                while let Some(ch) = state.peek() {
147                    if ch.is_ascii_digit() || ch == '.' { state.advance(1) } else { break }
148                }
149
150                state.add_token(TailwindTokenType::Number, start, state.get_position());
151                return true;
152            }
153        }
154
155        false
156    }
157
158    fn lex_punctuation<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
159        let start = state.get_position();
160        let rest = state.rest();
161
162        // Two-character operators
163        if rest.starts_with("{{") {
164            state.advance(2);
165            state.add_token(TailwindTokenType::DoubleLeftBrace, start, state.get_position());
166            return true;
167        }
168        if rest.starts_with("}}") {
169            state.advance(2);
170            state.add_token(TailwindTokenType::DoubleRightBrace, start, state.get_position());
171            return true;
172        }
173        if rest.starts_with("{%") {
174            state.advance(2);
175            state.add_token(TailwindTokenType::LeftBracePercent, start, state.get_position());
176            return true;
177        }
178        if rest.starts_with("%}") {
179            state.advance(2);
180            state.add_token(TailwindTokenType::PercentRightBrace, start, state.get_position());
181            return true;
182        }
183
184        // Single-character operators
185        if let Some(ch) = state.peek() {
186            let kind = match ch {
187                '{' => TailwindTokenType::LeftBrace,
188                '}' => TailwindTokenType::RightBrace,
189                '(' => TailwindTokenType::LeftParen,
190                ')' => TailwindTokenType::RightParen,
191                '[' => TailwindTokenType::LeftBracket,
192                ']' => TailwindTokenType::RightBracket,
193                ',' => TailwindTokenType::Comma,
194                '.' => TailwindTokenType::Dot,
195                ':' => TailwindTokenType::Colon,
196                ';' => TailwindTokenType::Semicolon,
197                '|' => TailwindTokenType::Pipe,
198                '=' => TailwindTokenType::Eq,
199                '+' => TailwindTokenType::Plus,
200                '-' => TailwindTokenType::Minus,
201                '*' => TailwindTokenType::Star,
202                '/' => TailwindTokenType::Slash,
203                '%' => TailwindTokenType::Percent,
204                '!' => TailwindTokenType::Bang,
205                '?' => TailwindTokenType::Question,
206                '<' => TailwindTokenType::Lt,
207                '>' => TailwindTokenType::Gt,
208                '&' => TailwindTokenType::Amp,
209                '^' => TailwindTokenType::Caret,
210                '~' => TailwindTokenType::Tilde,
211                _ => return false,
212            };
213
214            state.advance(1);
215            state.add_token(kind, start, state.get_position());
216            return true;
217        }
218
219        false
220    }
221
222    fn lex_identifier<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
223        let start = state.get_position();
224
225        if let Some(ch) = state.peek() {
226            if ch.is_ascii_alphabetic() || ch == '_' {
227                state.advance(ch.len_utf8());
228
229                while let Some(ch) = state.peek() {
230                    if ch.is_ascii_alphanumeric() || ch == '_' {
231                        state.advance(ch.len_utf8());
232                    }
233                    else {
234                        break;
235                    }
236                }
237
238                let end = state.get_position();
239                let text = state.get_text_in((start..end).into());
240
241                // Check if it's a boolean keyword
242                let kind = match text.as_ref() {
243                    "true" | "false" => TailwindTokenType::Boolean,
244                    _ => TailwindTokenType::Identifier,
245                };
246                state.add_token(kind, start, end);
247                return true;
248            }
249        }
250        false
251    }
252}
oak_tailwind/lexer/mod.rs

oak_tailwind/lexer/
mod.rs
