meow_meow_script/
tokenizer.rs

1use crate::ast::Span;
2use crate::token::{Token, TokenKind, TokenizeError, Unit};
3
4pub struct MeowMeowTokenizer<'a> {
5    input: &'a str,
6    bytes: &'a [u8],
7    idx: usize,
8}
9
10impl<'a> MeowMeowTokenizer<'a> {
11    pub fn new(input: &'a str) -> Self {
12        Self {
13            input,
14            bytes: input.as_bytes(),
15            idx: 0,
16        }
17    }
18
19    pub fn tokenize(mut self) -> Result<Vec<Token>, TokenizeError> {
20        let mut tokens = Vec::new();
21        loop {
22            let token = self.next_token()?;
23            let is_eof = matches!(token.kind, TokenKind::Eof);
24            tokens.push(token);
25            if is_eof {
26                break;
27            }
28        }
29        Ok(tokens)
30    }
31
32    fn next_token(&mut self) -> Result<Token, TokenizeError> {
33        self.skip_ws_and_comments()?;
34        let start = self.idx;
35        if self.idx >= self.bytes.len() {
36            return Ok(Token {
37                kind: TokenKind::Eof,
38                span: Span::new(self.idx, self.idx),
39            });
40        }
41
42        let b = self.bytes[self.idx];
43        let kind = match b {
44            b'{' => {
45                self.idx += 1;
46                TokenKind::LBrace
47            }
48            b'}' => {
49                self.idx += 1;
50                TokenKind::RBrace
51            }
52            b'(' => {
53                self.idx += 1;
54                TokenKind::LParen
55            }
56            b')' => {
57                self.idx += 1;
58                TokenKind::RParen
59            }
60            b'[' => {
61                self.idx += 1;
62                TokenKind::LBracket
63            }
64            b']' => {
65                self.idx += 1;
66                TokenKind::RBracket
67            }
68            b',' => {
69                self.idx += 1;
70                TokenKind::Comma
71            }
72            b'.' => {
73                self.idx += 1;
74                TokenKind::Dot
75            }
76            b';' => {
77                self.idx += 1;
78                TokenKind::Semicolon
79            }
80            b'+' => {
81                self.idx += 1;
82                TokenKind::Plus
83            }
84            b'-' => {
85                self.idx += 1;
86                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'>' {
87                    self.idx += 1;
88                    TokenKind::Arrow
89                } else {
90                    TokenKind::Minus
91                }
92            }
93            b'*' => {
94                self.idx += 1;
95                TokenKind::Star
96            }
97            b'/' => {
98                self.idx += 1;
99                TokenKind::Slash
100            }
101            b'%' => {
102                self.idx += 1;
103                TokenKind::Percent
104            }
105            b'=' => {
106                self.idx += 1;
107                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'=' {
108                    self.idx += 1;
109                    TokenKind::EqEq
110                } else {
111                    TokenKind::Eq
112                }
113            }
114            b'!' => {
115                self.idx += 1;
116                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'=' {
117                    self.idx += 1;
118                    TokenKind::BangEq
119                } else {
120                    TokenKind::Bang
121                }
122            }
123            b'<' => {
124                self.idx += 1;
125                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'=' {
126                    self.idx += 1;
127                    TokenKind::LtEq
128                } else {
129                    TokenKind::Lt
130                }
131            }
132            b'>' => {
133                self.idx += 1;
134                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'=' {
135                    self.idx += 1;
136                    TokenKind::GtEq
137                } else {
138                    TokenKind::Gt
139                }
140            }
141            b'&' => {
142                self.idx += 1;
143                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'&' {
144                    self.idx += 1;
145                    TokenKind::AmpAmp
146                } else {
147                    return Err(TokenizeError {
148                        message: "Expected '&&'".to_string(),
149                        span: Span::new(start, self.idx),
150                    });
151                }
152            }
153            b'|' => {
154                self.idx += 1;
155                if self.idx < self.bytes.len() && self.bytes[self.idx] == b'|' {
156                    self.idx += 1;
157                    TokenKind::PipePipe
158                } else if self.idx < self.bytes.len() && self.bytes[self.idx] == b'>' {
159                    self.idx += 1;
160                    TokenKind::PipeGt
161                } else {
162                    return Err(TokenizeError {
163                        message: "Expected '||' or '|>'".to_string(),
164                        span: Span::new(start, self.idx),
165                    });
166                }
167            }
168            b'"' => TokenKind::String(self.read_string()?),
169            b'0'..=b'9' => {
170                let n = self.read_number()?;
171                match self.try_read_unit_suffix() {
172                    Some(unit) => TokenKind::Dimension(n, unit),
173                    None => TokenKind::Number(n),
174                }
175            }
176            _ => {
177                if is_ident_start(b) {
178                    let ident = self.read_ident();
179                    match ident.as_str() {
180                        "let" => TokenKind::Let,
181                        "if" => TokenKind::If,
182                        "else" => TokenKind::Else,
183                        "return" => TokenKind::Return,
184                        "true" => TokenKind::True,
185                        "false" => TokenKind::False,
186                        "null" => TokenKind::Null,
187                        "fn" => TokenKind::Fn,
188                        "for" => TokenKind::For,
189                        "while" => TokenKind::While,
190                        "in" => TokenKind::In,
191                        "break" => TokenKind::Break,
192                        "continue" => TokenKind::Continue,
193                        "export" => TokenKind::Export,
194                        "import" => TokenKind::Import,
195                        "from" => TokenKind::From,
196                        "as" => TokenKind::As,
197                        _ => TokenKind::Ident(ident),
198                    }
199                } else {
200                    return Err(TokenizeError {
201                        message: format!("Unexpected character: {}", self.current_char_debug()),
202                        span: Span::new(self.idx, self.idx + 1),
203                    });
204                }
205            }
206        };
207
208        Ok(Token {
209            kind,
210            span: Span::new(start, self.idx),
211        })
212    }
213
214    fn skip_ws_and_comments(&mut self) -> Result<(), TokenizeError> {
215        loop {
216            while self.idx < self.bytes.len() {
217                match self.bytes[self.idx] {
218                    b' ' | b'\t' | b'\n' | b'\r' => self.idx += 1,
219                    _ => break,
220                }
221            }
222
223            // line comment
224            if self.peek2() == Some((b'/', b'/')) {
225                self.idx += 2;
226                while self.idx < self.bytes.len() && self.bytes[self.idx] != b'\n' {
227                    self.idx += 1;
228                }
229                continue;
230            }
231
232            // block comment
233            if self.peek2() == Some((b'/', b'*')) {
234                let start = self.idx;
235                self.idx += 2;
236                while self.idx + 1 < self.bytes.len() {
237                    if self.bytes[self.idx] == b'*' && self.bytes[self.idx + 1] == b'/' {
238                        self.idx += 2;
239                        break;
240                    }
241                    self.idx += 1;
242                }
243                if self.idx >= self.bytes.len() {
244                    return Err(TokenizeError {
245                        message: "Unterminated block comment".to_string(),
246                        span: Span::new(start, self.bytes.len()),
247                    });
248                }
249                continue;
250            }
251
252            break;
253        }
254
255        Ok(())
256    }
257
258    fn read_ident(&mut self) -> String {
259        let start = self.idx;
260        self.idx += 1;
261        while self.idx < self.bytes.len() && is_ident_continue(self.bytes[self.idx]) {
262            self.idx += 1;
263        }
264        self.input[start..self.idx].to_string()
265    }
266
267    fn read_string(&mut self) -> Result<String, TokenizeError> {
268        let start = self.idx;
269        // opening quote
270        self.idx += 1;
271        let mut out = String::new();
272        while self.idx < self.bytes.len() {
273            let b = self.bytes[self.idx];
274            match b {
275                b'"' => {
276                    self.idx += 1;
277                    return Ok(out);
278                }
279                b'\\' => {
280                    self.idx += 1;
281                    if self.idx >= self.bytes.len() {
282                        break;
283                    }
284                    let esc = self.bytes[self.idx];
285                    self.idx += 1;
286                    match esc {
287                        b'"' => out.push('"'),
288                        b'\\' => out.push('\\'),
289                        b'n' => out.push('\n'),
290                        b'r' => out.push('\r'),
291                        b't' => out.push('\t'),
292                        _ => {
293                            return Err(TokenizeError {
294                                message: format!("Unsupported escape: \\\\{}", esc as char),
295                                span: Span::new(self.idx - 2, self.idx),
296                            });
297                        }
298                    }
299                }
300                _ => {
301                    // UTF-8: take the next char boundary using str methods
302                    let s = &self.input[self.idx..];
303                    if let Some(ch) = s.chars().next() {
304                        out.push(ch);
305                        self.idx += ch.len_utf8();
306                    } else {
307                        break;
308                    }
309                }
310            }
311        }
312
313        Err(TokenizeError {
314            message: "Unterminated string literal".to_string(),
315            span: Span::new(start, self.idx),
316        })
317    }
318
319    fn read_number(&mut self) -> Result<f64, TokenizeError> {
320        let start = self.idx;
321        while self.idx < self.bytes.len() {
322            match self.bytes[self.idx] {
323                b'0'..=b'9' => self.idx += 1,
324                _ => break,
325            }
326        }
327        if self.idx < self.bytes.len() && self.bytes[self.idx] == b'.' {
328            self.idx += 1;
329            while self.idx < self.bytes.len() {
330                match self.bytes[self.idx] {
331                    b'0'..=b'9' => self.idx += 1,
332                    _ => break,
333                }
334            }
335        }
336
337        let s = &self.input[start..self.idx];
338        s.parse::<f64>().map_err(|_| TokenizeError {
339            message: format!("Invalid number literal: {s}"),
340            span: Span::new(start, self.idx),
341        })
342    }
343
344    /// Try to consume a unit suffix attached (no whitespace) to a numeric
345    /// literal. Recognized: `%`, `gu`, `deg`, `rad`. Returns `None` if the
346    /// next character isn't part of a recognized suffix — caller falls back
347    /// to a bare `Number` token.
348    fn try_read_unit_suffix(&mut self) -> Option<Unit> {
349        if self.idx >= self.bytes.len() {
350            return None;
351        }
352        if self.bytes[self.idx] == b'%' {
353            self.idx += 1;
354            return Some(Unit::Percent);
355        }
356        // Letter-prefixed suffixes: peek without consuming, only commit on match.
357        let start = self.idx;
358        let mut end = start;
359        while end < self.bytes.len() && is_ident_continue(self.bytes[end]) {
360            end += 1;
361        }
362        if end == start {
363            return None;
364        }
365        let unit = match &self.input[start..end] {
366            "gu" => Unit::GlyphUnits,
367            "wu" => Unit::WorldUnits,
368            "deg" => Unit::Degrees,
369            "rad" => Unit::Radians,
370            _ => return None,
371        };
372        self.idx = end;
373        Some(unit)
374    }
375
376    fn peek2(&self) -> Option<(u8, u8)> {
377        if self.idx + 1 >= self.bytes.len() {
378            None
379        } else {
380            Some((self.bytes[self.idx], self.bytes[self.idx + 1]))
381        }
382    }
383
384    fn current_char_debug(&self) -> String {
385        if self.idx >= self.bytes.len() {
386            return "<eof>".to_string();
387        }
388        let s = &self.input[self.idx..];
389        s.chars()
390            .next()
391            .map(|c| c.to_string())
392            .unwrap_or("<invalid>".to_string())
393    }
394}
395
396fn is_ident_start(b: u8) -> bool {
397    matches!(b, b'a'..=b'z' | b'A'..=b'Z' | b'_')
398}
399
400fn is_ident_continue(b: u8) -> bool {
401    is_ident_start(b) || matches!(b, b'0'..=b'9')
402}
meow_meow_script/tokenizer.rs

meow_meow_script/
tokenizer.rs
