oak_scss/lexer/
mod.rs

1#![doc = include_str!("readme.md")]
2/// Token type definitions for the SCSS lexer.
3/// Token type definitions for SCSS.
4pub mod token_type;
5
6use crate::{language::ScssLanguage, lexer::token_type::ScssTokenType};
7use oak_core::{
8    Lexer, LexerState, OakError, Source, TextEdit,
9    lexer::{CommentConfig, LexOutput, LexerCache, StringConfig, WhitespaceConfig},
10};
11use std::sync::LazyLock;
12
13pub(crate) type State<'a, S> = LexerState<'a, S, ScssLanguage>;
14
15static SCSS_WHITESPACE: LazyLock<WhitespaceConfig> = LazyLock::new(|| WhitespaceConfig { unicode_whitespace: true });
16static SCSS_COMMENT: LazyLock<CommentConfig> = LazyLock::new(|| CommentConfig { line_marker: "//", block_start: "/*", block_end: "*/", nested_blocks: true });
17static SCSS_STRING: LazyLock<StringConfig> = LazyLock::new(|| StringConfig { quotes: &['"'], escape: Some('\\') });
18
19/// Lexer for the SCSS language.
20#[derive(Debug, Clone)]
21pub struct ScssLexer<'config> {
22    config: &'config ScssLanguage,
23}
24
25impl<'config> Lexer<ScssLanguage> for ScssLexer<'config> {
26    /// Tokenizes the input source text.
27    fn lex<'a, S: Source + ?Sized>(&self, text: &S, _edits: &[TextEdit], cache: &'a mut impl LexerCache<ScssLanguage>) -> LexOutput<ScssLanguage> {
28        let mut state = LexerState::new(text);
29        let result = self.run(&mut state);
30        if result.is_ok() {
31            state.add_eof();
32        }
33        state.finish_with_cache(result, cache)
34    }
35}
36
37impl<'config> ScssLexer<'config> {
38    /// Creates a new `ScssLexer` with the given configuration.
39    pub fn new(config: &'config ScssLanguage) -> Self {
40        Self { config }
41    }
42
43    /// Main lexer loop that tokenizes the source text.
44    fn run<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> Result<(), OakError> {
45        while state.not_at_end() {
46            let safe_point = state.get_position();
47
48            if self.skip_whitespace(state) {
49                continue;
50            }
51
52            if self.lex_newline(state) {
53                continue;
54            }
55
56            if self.skip_comment(state) {
57                continue;
58            }
59
60            if self.lex_string_literal(state) {
61                continue;
62            }
63
64            if self.lex_number_literal(state) {
65                continue;
66            }
67
68            if self.lex_identifier_or_keyword(state) {
69                continue;
70            }
71
72            if self.lex_operators(state) {
73                continue;
74            }
75
76            if self.lex_single_char_tokens(state) {
77                continue;
78            }
79
80            // Error handling: if no rule matches, skip current character and mark as error
81            let start_pos = state.get_position();
82            if let Some(ch) = state.peek() {
83                state.advance(ch.len_utf8());
84                state.add_token(ScssTokenType::Error, start_pos, state.get_position());
85            }
86
87            state.advance_if_dead_lock(safe_point)
88        }
89
90        Ok(())
91    }
92
93    /// Skips whitespace characters.
94    fn skip_whitespace<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
95        SCSS_WHITESPACE.scan(state, ScssTokenType::Whitespace)
96    }
97
98    /// Handles newlines.
99    fn lex_newline<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
100        let start_pos = state.get_position();
101
102        if let Some('\n') = state.peek() {
103            state.advance(1);
104            state.add_token(ScssTokenType::Newline, start_pos, state.get_position());
105            true
106        }
107        else if let Some('\r') = state.peek() {
108            state.advance(1);
109            if let Some('\n') = state.peek() {
110                state.advance(1)
111            }
112            state.add_token(ScssTokenType::Newline, start_pos, state.get_position());
113            true
114        }
115        else {
116            false
117        }
118    }
119
120    /// Skips comments.
121    fn skip_comment<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
122        SCSS_COMMENT.scan(state, ScssTokenType::Comment, ScssTokenType::Comment)
123    }
124
125    /// Lexes string literals.
126    fn lex_string_literal<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
127        SCSS_STRING.scan(state, ScssTokenType::StringLiteral)
128    }
129
130    /// Lexes number literals.
131    fn lex_number_literal<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
132        let start = state.get_position();
133
134        if let Some(first_char) = state.peek() {
135            if first_char.is_ascii_digit() {
136                state.advance(first_char.len_utf8());
137
138                // Continue with digits
139                while let Some(ch) = state.peek() {
140                    if ch.is_ascii_digit() { state.advance(ch.len_utf8()) } else { break }
141                }
142
143                // Handle decimal point
144                if state.peek() == Some('.') && state.peek_next_n(1).map_or(false, |c| c.is_ascii_digit()) {
145                    state.advance(1); // consume '.'
146                    while let Some(ch) = state.peek() {
147                        if ch.is_ascii_digit() { state.advance(ch.len_utf8()) } else { break }
148                    }
149                }
150
151                state.add_token(ScssTokenType::IntegerLiteral, start, state.get_position());
152                return true;
153            }
154        }
155        false
156    }
157
158    /// Lexes identifiers or keywords.
159    fn lex_identifier_or_keyword<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
160        let start = state.get_position();
161        let text = state.source().get_text_from(start);
162
163        if let Some(first_char) = text.chars().next() {
164            if first_char.is_alphabetic() || first_char == '_' || first_char == '$' {
165                let mut len = first_char.len_utf8();
166
167                let mut chars = text.chars().skip(1);
168                while let Some(ch) = chars.next() {
169                    if ch.is_alphanumeric() || ch == '_' || ch == '-' {
170                        len += ch.len_utf8();
171                    }
172                    else {
173                        break;
174                    }
175                }
176
177                let word = &text[..len];
178                let kind = self.keyword_kind(word).unwrap_or(ScssTokenType::Identifier);
179                state.advance(len);
180                state.add_token(kind, start, state.get_position());
181                return true;
182            }
183        }
184        false
185    }
186
187    /// Lexes operators.
188    fn lex_operators<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
189        let start = state.get_position();
190        let text = state.source().get_text_from(start);
191
192        // Two-character operators
193        if text.len() >= 2 {
194            let two_char = &text[..2];
195            if let Some(kind) = self.operator_kind(two_char) {
196                state.advance(2);
197                state.add_token(kind, start, state.get_position());
198                return true;
199            }
200        }
201
202        // Single-character operators
203        if let Some(first_char) = text.chars().next() {
204            let single_char = &text[..first_char.len_utf8()];
205            if let Some(kind) = self.operator_kind(single_char) {
206                state.advance(first_char.len_utf8());
207                state.add_token(kind, start, state.get_position());
208                return true;
209            }
210        }
211
212        false
213    }
214
215    /// Lexes single character tokens.
216    fn lex_single_char_tokens<'a, S: Source + ?Sized>(&self, state: &mut State<'a, S>) -> bool {
217        let start = state.get_position();
218        let text = state.source().get_text_from(start);
219
220        if let Some(first_char) = text.chars().next() {
221            let single_char = &text[..first_char.len_utf8()];
222            if let Some(kind) = self.single_char_kind(single_char) {
223                state.advance(first_char.len_utf8());
224                state.add_token(kind, start, state.get_position());
225                return true;
226            }
227        }
228
229        false
230    }
231
232    /// Returns the syntax kind for a given keyword.
233    fn keyword_kind(&self, text: &str) -> Option<ScssTokenType> {
234        match text {
235            "import" => Some(ScssTokenType::Import),
236            "include" => Some(ScssTokenType::Include),
237            "mixin" => Some(ScssTokenType::Mixin),
238            "function" => Some(ScssTokenType::Function),
239            "return" => Some(ScssTokenType::Return),
240            "if" => Some(ScssTokenType::If),
241            "else" => Some(ScssTokenType::Else),
242            "for" => Some(ScssTokenType::For),
243            "while" => Some(ScssTokenType::While),
244            "each" => Some(ScssTokenType::Each),
245            "in" => Some(ScssTokenType::In),
246            "true" => Some(ScssTokenType::True),
247            "false" => Some(ScssTokenType::False),
248            "null" => Some(ScssTokenType::Null),
249            _ => None,
250        }
251    }
252
253    fn operator_kind(&self, text: &str) -> Option<ScssTokenType> {
254        match text {
255            "==" => Some(ScssTokenType::EqEq),
256            "!=" => Some(ScssTokenType::Ne),
257            "<=" => Some(ScssTokenType::Le),
258            ">=" => Some(ScssTokenType::Ge),
259            "&&" => Some(ScssTokenType::AndAnd),
260            "||" => Some(ScssTokenType::OrOr),
261            "=" => Some(ScssTokenType::Eq),
262            "<" => Some(ScssTokenType::Lt),
263            ">" => Some(ScssTokenType::Gt),
264            "&" => Some(ScssTokenType::And),
265            "|" => Some(ScssTokenType::Or),
266            "^" => Some(ScssTokenType::Xor),
267            "+" => Some(ScssTokenType::Plus),
268            "-" => Some(ScssTokenType::Minus),
269            "*" => Some(ScssTokenType::Star),
270            "/" => Some(ScssTokenType::Slash),
271            "%" => Some(ScssTokenType::Percent),
272            "!" => Some(ScssTokenType::Bang),
273            _ => None,
274        }
275    }
276
277    fn single_char_kind(&self, text: &str) -> Option<ScssTokenType> {
278        match text {
279            "(" => Some(ScssTokenType::LeftParen),
280            ")" => Some(ScssTokenType::RightParen),
281            "{" => Some(ScssTokenType::LeftBrace),
282            "}" => Some(ScssTokenType::RightBrace),
283            "[" => Some(ScssTokenType::LeftBracket),
284            "]" => Some(ScssTokenType::RightBracket),
285            ";" => Some(ScssTokenType::Semicolon),
286            ":" => Some(ScssTokenType::Colon),
287            "," => Some(ScssTokenType::Comma),
288            "." => Some(ScssTokenType::Dot),
289            "#" => Some(ScssTokenType::Hash),
290            "@" => Some(ScssTokenType::At),
291            "$" => Some(ScssTokenType::Dollar),
292            _ => None,
293        }
294    }
295}
oak_scss/lexer/mod.rs

oak_scss/lexer/
mod.rs
