newter_compiler/
lexer.rs

1//! Lexer for the Newt UI language.
2
3use crate::error::{NewtError, Span};
4use std::fmt;
5
6/// A segment of an interpolated string at the token level.
7#[derive(Clone, Debug, PartialEq)]
8pub enum InterpPart {
9    /// Literal text between interpolation braces.
10    Literal(String),
11    /// Raw source text of an expression inside `{…}`.
12    ExprSource(String),
13}
14
15#[derive(Clone, Debug, PartialEq)]
16pub enum TokenKind {
17    // Literals
18    Number(f64),
19    String(String),
20    /// String containing `{expr}` interpolation segments.
21    InterpolatedString(Vec<InterpPart>),
22    Ident(String),
23    True,
24    False,
25    HexColor(u8, u8, u8, u8),
26
27    // Keywords (language)
28    Let,
29    State,
30    Component,
31    Screen,
32    If,
33    Else,
34    For,
35    In,
36    Theme,
37    Use,
38    Import,
39
40    // Element types (UI) — sections and layout
41    Header,
42    Footer,
43    Container,
44    Sidebar,
45    Section,
46    Box,
47    Text,
48    Row,
49    Column,
50    Grid,
51    Stack,
52    Center,
53    Spacer,
54    Image,
55    Button,
56    Input,
57    Card,
58    Widget,
59    // Navigational
60    Accordion,
61    Bento,
62    Breadcrumb,
63    Hamburger,
64    Kebab,
65    Meatballs,
66    Doner,
67    Tabs,
68    Pagination,
69    LinkList,
70    Nav,
71    // Input and forms
72    Password,
73    Search,
74    Checkbox,
75    Radio,
76    Dropdown,
77    Combobox,
78    Multiselect,
79    DatePicker,
80    Picker,
81    Slider,
82    Stepper,
83    Toggle,
84    Form,
85    // Feedback and overlay
86    Modal,
87    ConfirmDialog,
88    Toast,
89    Notification,
90    Alert,
91    MessageBox,
92    Tooltip,
93    Loader,
94    ProgressBar,
95    Badge,
96    // Display and content
97    Icon,
98    Tag,
99    Comment,
100    Feed,
101    Carousel,
102    Chart,
103
104    // New elements (v0.2)
105    Table,
106    Avatar,
107    Skeleton,
108    Drawer,
109    Select,
110    Textarea,
111    Popover,
112    Separator,
113    Timeline,
114    Rating,
115    FileUpload,
116    ColorPicker,
117    TreeView,
118    CommandPalette,
119    Splitter,
120
121    // Property names (common)
122    Width,
123    Height,
124    Fill,
125    Stroke,
126    Radius,
127    Padding,
128    Gap,
129    Grow,
130    Shrink,
131    Align,
132    Justify,
133    Direction,
134    FontSize,
135    FontWeight,
136    Shadow,
137
138    // Delimiters
139    LeftBrace,
140    RightBrace,
141    LeftParen,
142    RightParen,
143    LeftBracket,
144    RightBracket,
145    Comma,
146    Dot,
147    Semicolon,
148    Colon,
149    Arrow, // ->
150
151    // Operators
152    Eq,       // =
153    Plus,
154    Minus,
155    Star,
156    Slash,
157    Percent,
158    EqEq,
159    NotEq,
160    Lt,
161    Le,
162    Gt,
163    Ge,
164    And,
165    Or,
166    Not,
167
168    Eof,
169}
170
171impl fmt::Display for TokenKind {
172    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
173        match self {
174            TokenKind::Number(n) => write!(f, "{}", n),
175            TokenKind::String(s) => write!(f, "\"{}\"", s),
176            TokenKind::InterpolatedString(_) => write!(f, "\"…{{…}}…\""),
177            TokenKind::Ident(s) => write!(f, "{}", s),
178            TokenKind::True => write!(f, "true"),
179            TokenKind::False => write!(f, "false"),
180            TokenKind::HexColor(r, g, b, a) => write!(f, "#{:02x}{:02x}{:02x}{:02x}", r, g, b, a),
181            TokenKind::Let => write!(f, "let"),
182            TokenKind::State => write!(f, "state"),
183            TokenKind::Component => write!(f, "component"),
184            TokenKind::Screen => write!(f, "screen"),
185            TokenKind::If => write!(f, "if"),
186            TokenKind::Else => write!(f, "else"),
187            TokenKind::For => write!(f, "for"),
188            TokenKind::In => write!(f, "in"),
189            TokenKind::Theme => write!(f, "theme"),
190            TokenKind::Use => write!(f, "use"),
191            TokenKind::Import => write!(f, "import"),
192            TokenKind::Header => write!(f, "header"),
193            TokenKind::Footer => write!(f, "footer"),
194            TokenKind::Container => write!(f, "container"),
195            TokenKind::Sidebar => write!(f, "sidebar"),
196            TokenKind::Box => write!(f, "box"),
197            TokenKind::Text => write!(f, "text"),
198            TokenKind::Row => write!(f, "row"),
199            TokenKind::Column => write!(f, "column"),
200            TokenKind::Grid => write!(f, "grid"),
201            TokenKind::Stack => write!(f, "stack"),
202            TokenKind::Center => write!(f, "center"),
203            TokenKind::Spacer => write!(f, "spacer"),
204            TokenKind::Image => write!(f, "image"),
205            TokenKind::Button => write!(f, "button"),
206            TokenKind::Input => write!(f, "input"),
207            TokenKind::Card => write!(f, "card"),
208            TokenKind::Section => write!(f, "section"),
209            TokenKind::Widget => write!(f, "widget"),
210            TokenKind::Accordion => write!(f, "accordion"),
211            TokenKind::Bento => write!(f, "bento"),
212            TokenKind::Breadcrumb => write!(f, "breadcrumb"),
213            TokenKind::Hamburger => write!(f, "hamburger"),
214            TokenKind::Kebab => write!(f, "kebab"),
215            TokenKind::Meatballs => write!(f, "meatballs"),
216            TokenKind::Doner => write!(f, "doner"),
217            TokenKind::Tabs => write!(f, "tabs"),
218            TokenKind::Pagination => write!(f, "pagination"),
219            TokenKind::LinkList => write!(f, "linkList"),
220            TokenKind::Nav => write!(f, "nav"),
221            TokenKind::Password => write!(f, "password"),
222            TokenKind::Search => write!(f, "search"),
223            TokenKind::Checkbox => write!(f, "checkbox"),
224            TokenKind::Radio => write!(f, "radio"),
225            TokenKind::Dropdown => write!(f, "dropdown"),
226            TokenKind::Combobox => write!(f, "combobox"),
227            TokenKind::Multiselect => write!(f, "multiselect"),
228            TokenKind::DatePicker => write!(f, "datePicker"),
229            TokenKind::Picker => write!(f, "picker"),
230            TokenKind::Slider => write!(f, "slider"),
231            TokenKind::Stepper => write!(f, "stepper"),
232            TokenKind::Toggle => write!(f, "toggle"),
233            TokenKind::Form => write!(f, "form"),
234            TokenKind::Modal => write!(f, "modal"),
235            TokenKind::ConfirmDialog => write!(f, "confirmDialog"),
236            TokenKind::Toast => write!(f, "toast"),
237            TokenKind::Notification => write!(f, "notification"),
238            TokenKind::Alert => write!(f, "alert"),
239            TokenKind::MessageBox => write!(f, "messageBox"),
240            TokenKind::Tooltip => write!(f, "tooltip"),
241            TokenKind::Loader => write!(f, "loader"),
242            TokenKind::ProgressBar => write!(f, "progressBar"),
243            TokenKind::Badge => write!(f, "badge"),
244            TokenKind::Icon => write!(f, "icon"),
245            TokenKind::Tag => write!(f, "tag"),
246            TokenKind::Comment => write!(f, "comment"),
247            TokenKind::Feed => write!(f, "feed"),
248            TokenKind::Carousel => write!(f, "carousel"),
249            TokenKind::Chart => write!(f, "chart"),
250            TokenKind::Table => write!(f, "table"),
251            TokenKind::Avatar => write!(f, "avatar"),
252            TokenKind::Skeleton => write!(f, "skeleton"),
253            TokenKind::Drawer => write!(f, "drawer"),
254            TokenKind::Select => write!(f, "select"),
255            TokenKind::Textarea => write!(f, "textarea"),
256            TokenKind::Popover => write!(f, "popover"),
257            TokenKind::Separator => write!(f, "separator"),
258            TokenKind::Timeline => write!(f, "timeline"),
259            TokenKind::Rating => write!(f, "rating"),
260            TokenKind::FileUpload => write!(f, "fileUpload"),
261            TokenKind::ColorPicker => write!(f, "colorPicker"),
262            TokenKind::TreeView => write!(f, "treeView"),
263            TokenKind::CommandPalette => write!(f, "commandPalette"),
264            TokenKind::Splitter => write!(f, "splitter"),
265            TokenKind::Width => write!(f, "width"),
266            TokenKind::Height => write!(f, "height"),
267            TokenKind::Fill => write!(f, "fill"),
268            TokenKind::Stroke => write!(f, "stroke"),
269            TokenKind::Radius => write!(f, "radius"),
270            TokenKind::Padding => write!(f, "padding"),
271            TokenKind::Gap => write!(f, "gap"),
272            TokenKind::Grow => write!(f, "grow"),
273            TokenKind::Shrink => write!(f, "shrink"),
274            TokenKind::Align => write!(f, "align"),
275            TokenKind::Justify => write!(f, "justify"),
276            TokenKind::Direction => write!(f, "direction"),
277            TokenKind::FontSize => write!(f, "fontSize"),
278            TokenKind::FontWeight => write!(f, "fontWeight"),
279            TokenKind::Shadow => write!(f, "shadow"),
280            TokenKind::LeftBrace => write!(f, "{{"),
281            TokenKind::RightBrace => write!(f, "}}"),
282            TokenKind::LeftParen => write!(f, "("),
283            TokenKind::RightParen => write!(f, ")"),
284            TokenKind::LeftBracket => write!(f, "["),
285            TokenKind::RightBracket => write!(f, "]"),
286            TokenKind::Comma => write!(f, ","),
287            TokenKind::Dot => write!(f, "."),
288            TokenKind::Semicolon => write!(f, ";"),
289            TokenKind::Colon => write!(f, ":"),
290            TokenKind::Arrow => write!(f, "->"),
291            TokenKind::Eq => write!(f, "="),
292            TokenKind::Plus => write!(f, "+"),
293            TokenKind::Minus => write!(f, "-"),
294            TokenKind::Star => write!(f, "*"),
295            TokenKind::Slash => write!(f, "/"),
296            TokenKind::Percent => write!(f, "%"),
297            TokenKind::EqEq => write!(f, "=="),
298            TokenKind::NotEq => write!(f, "!="),
299            TokenKind::Lt => write!(f, "<"),
300            TokenKind::Le => write!(f, "<="),
301            TokenKind::Gt => write!(f, ">"),
302            TokenKind::Ge => write!(f, ">="),
303            TokenKind::And => write!(f, "&&"),
304            TokenKind::Or => write!(f, "||"),
305            TokenKind::Not => write!(f, "!"),
306            TokenKind::Eof => write!(f, "<eof>"),
307        }
308    }
309}
310
311/// Token category for syntax highlighting (keyword, string, number, etc.).
312#[derive(Clone, Copy, Debug, PartialEq, Eq)]
313pub enum TokenCategory {
314    Keyword,
315    String,
316    Number,
317    Color,
318    Ident,
319    Comment,
320    Delimiter,
321    Eof,
322}
323
324impl TokenKind {
325    pub fn category(&self) -> TokenCategory {
326        match self {
327            TokenKind::Number(_) => TokenCategory::Number,
328            TokenKind::String(_) | TokenKind::InterpolatedString(_) => TokenCategory::String,
329            TokenKind::Ident(_) => TokenCategory::Ident,
330            TokenKind::True | TokenKind::False => TokenCategory::Ident,
331            TokenKind::HexColor(_, _, _, _) => TokenCategory::Color,
332            TokenKind::Let | TokenKind::State | TokenKind::Component | TokenKind::Screen | TokenKind::If
333            | TokenKind::Else | TokenKind::For | TokenKind::In | TokenKind::Theme
334            | TokenKind::Use | TokenKind::Import => TokenCategory::Keyword,
335            TokenKind::Header | TokenKind::Footer | TokenKind::Container | TokenKind::Sidebar
336            | TokenKind::Section | TokenKind::Box | TokenKind::Text | TokenKind::Row
337            | TokenKind::Column | TokenKind::Grid | TokenKind::Stack | TokenKind::Center
338            | TokenKind::Spacer | TokenKind::Image | TokenKind::Button | TokenKind::Input
339            | TokenKind::Card | TokenKind::Widget => TokenCategory::Keyword,
340            TokenKind::Accordion | TokenKind::Bento | TokenKind::Breadcrumb | TokenKind::Hamburger
341            | TokenKind::Kebab | TokenKind::Meatballs | TokenKind::Doner | TokenKind::Tabs
342            | TokenKind::Pagination | TokenKind::LinkList | TokenKind::Nav | TokenKind::Password
343            | TokenKind::Search | TokenKind::Checkbox | TokenKind::Radio | TokenKind::Dropdown
344            | TokenKind::Combobox | TokenKind::Multiselect | TokenKind::DatePicker | TokenKind::Picker
345            | TokenKind::Slider | TokenKind::Stepper | TokenKind::Toggle | TokenKind::Form
346            | TokenKind::Modal | TokenKind::ConfirmDialog | TokenKind::Toast | TokenKind::Notification
347            | TokenKind::Alert | TokenKind::MessageBox | TokenKind::Tooltip | TokenKind::Loader
348            | TokenKind::ProgressBar | TokenKind::Badge | TokenKind::Icon | TokenKind::Tag
349            | TokenKind::Comment | TokenKind::Feed | TokenKind::Carousel | TokenKind::Chart
350            | TokenKind::Table | TokenKind::Avatar | TokenKind::Skeleton | TokenKind::Drawer
351            | TokenKind::Select | TokenKind::Textarea | TokenKind::Popover | TokenKind::Separator
352            | TokenKind::Timeline | TokenKind::Rating | TokenKind::FileUpload | TokenKind::ColorPicker
353            | TokenKind::TreeView | TokenKind::CommandPalette | TokenKind::Splitter => TokenCategory::Keyword,
354            TokenKind::Width | TokenKind::Height | TokenKind::Fill | TokenKind::Stroke
355            | TokenKind::Radius | TokenKind::Padding | TokenKind::Gap | TokenKind::Grow
356            | TokenKind::Shrink | TokenKind::Align | TokenKind::Justify | TokenKind::Direction
357            | TokenKind::FontSize | TokenKind::FontWeight | TokenKind::Shadow => TokenCategory::Ident,
358            TokenKind::Eof => TokenCategory::Eof,
359            _ => TokenCategory::Delimiter,
360        }
361    }
362}
363
364#[derive(Clone, Debug)]
365pub struct Token {
366    pub kind: TokenKind,
367    pub span: Span,
368}
369
370pub struct Lexer<'a> {
371    source: &'a str,
372    pos: usize,
373    line: u32,
374    column: u32,
375}
376
377impl<'a> Lexer<'a> {
378    pub fn new(source: &'a str, _path: Option<&'a str>) -> Self {
379        Self {
380            source,
381            pos: 0,
382            line: 1,
383            column: 1,
384        }
385    }
386
387    fn peek(&self) -> Option<char> {
388        self.source[self.pos..].chars().next()
389    }
390
391    fn peek2(&self) -> (Option<char>, Option<char>) {
392        let mut it = self.source[self.pos..].chars();
393        (it.next(), it.next())
394    }
395
396    fn advance(&mut self) -> Option<char> {
397        let ch = self.peek()?;
398        if ch == '\n' {
399            self.line += 1;
400            self.column = 1;
401        } else {
402            self.column += 1;
403        }
404        self.pos += ch.len_utf8();
405        Some(ch)
406    }
407
408    fn span_start(&self) -> (usize, u32, u32) {
409        (self.pos, self.line, self.column)
410    }
411
412    fn make_span(&self, start: usize, start_line: u32, start_col: u32) -> Span {
413        Span::new(start, self.pos, start_line, start_col)
414    }
415
416    fn skip_whitespace_and_comments(&mut self) {
417        loop {
418            match self.peek() {
419                Some(' ' | '\t' | '\r' | '\n') => {
420                    self.advance();
421                }
422                Some('/') if self.peek2().1 == Some('/') => {
423                    self.advance();
424                    self.advance();
425                    while self.peek().map(|c| c != '\n').unwrap_or(false) {
426                        self.advance();
427                    }
428                }
429                _ => break,
430            }
431        }
432    }
433
434    fn read_string(&mut self) -> Result<Token, NewtError> {
435        let (start, line, col) = self.span_start();
436        self.advance(); // opening "
437        let mut parts: Vec<InterpPart> = Vec::new();
438        let mut buf = String::new();
439        let mut has_interp = false;
440
441        while let Some(c) = self.peek() {
442            if c == '"' {
443                self.advance();
444                if !has_interp {
445                    return Ok(Token {
446                        kind: TokenKind::String(buf),
447                        span: self.make_span(start, line, col),
448                    });
449                }
450                if !buf.is_empty() {
451                    parts.push(InterpPart::Literal(buf));
452                }
453                return Ok(Token {
454                    kind: TokenKind::InterpolatedString(parts),
455                    span: self.make_span(start, line, col),
456                });
457            }
458            if c == '\\' {
459                self.advance();
460                match self.advance() {
461                    Some('n') => buf.push('\n'),
462                    Some('t') => buf.push('\t'),
463                    Some('r') => buf.push('\r'),
464                    Some('"') => buf.push('"'),
465                    Some('\\') => buf.push('\\'),
466                    Some('{') => buf.push('{'),
467                    Some('}') => buf.push('}'),
468                    Some(o) => buf.push(o),
469                    None => break,
470                }
471                continue;
472            }
473            if c == '{' {
474                has_interp = true;
475                self.advance(); // consume {
476                if !buf.is_empty() {
477                    parts.push(InterpPart::Literal(std::mem::take(&mut buf)));
478                }
479                // Collect expression source until matching }
480                let mut depth = 1u32;
481                let mut expr_src = String::new();
482                while let Some(ec) = self.peek() {
483                    if ec == '{' {
484                        depth += 1;
485                        expr_src.push(ec);
486                        self.advance();
487                    } else if ec == '}' {
488                        depth -= 1;
489                        if depth == 0 {
490                            self.advance(); // consume closing }
491                            break;
492                        }
493                        expr_src.push(ec);
494                        self.advance();
495                    } else if ec == '"' {
496                        // String literal inside interpolation — consume it whole
497                        expr_src.push(ec);
498                        self.advance();
499                        while let Some(sc) = self.peek() {
500                            expr_src.push(sc);
501                            self.advance();
502                            if sc == '"' { break; }
503                            if sc == '\\' {
504                                if let Some(esc) = self.peek() {
505                                    expr_src.push(esc);
506                                    self.advance();
507                                }
508                            }
509                        }
510                    } else {
511                        expr_src.push(ec);
512                        self.advance();
513                    }
514                }
515                if depth != 0 {
516                    return Err(NewtError::lexer(
517                        self.make_span(start, line, col),
518                        "unterminated interpolation brace in string",
519                    ));
520                }
521                parts.push(InterpPart::ExprSource(expr_src));
522                continue;
523            }
524            buf.push(c);
525            self.advance();
526        }
527        Err(NewtError::lexer(
528            self.make_span(start, line, col),
529            "unterminated string",
530        ))
531    }
532
533    fn read_hex_color(&mut self) -> Result<Token, NewtError> {
534        let (start, line, col) = self.span_start();
535        self.advance(); // #
536        let mut s = String::new();
537        while let Some(c) = self.peek() {
538            if c.is_ascii_hexdigit() {
539                s.push(c);
540                self.advance();
541            } else {
542                break;
543            }
544        }
545        if s.len() == 6 {
546            let r = u8::from_str_radix(&s[0..2], 16).unwrap_or(0);
547            let g = u8::from_str_radix(&s[2..4], 16).unwrap_or(0);
548            let b = u8::from_str_radix(&s[4..6], 16).unwrap_or(0);
549            Ok(Token {
550                kind: TokenKind::HexColor(r, g, b, 255),
551                span: self.make_span(start, line, col),
552            })
553        } else if s.len() == 8 {
554            let r = u8::from_str_radix(&s[0..2], 16).unwrap_or(0);
555            let g = u8::from_str_radix(&s[2..4], 16).unwrap_or(0);
556            let b = u8::from_str_radix(&s[4..6], 16).unwrap_or(0);
557            let a = u8::from_str_radix(&s[6..8], 16).unwrap_or(255);
558            Ok(Token {
559                kind: TokenKind::HexColor(r, g, b, a),
560                span: self.make_span(start, line, col),
561            })
562        } else {
563            Err(NewtError::lexer(
564                self.make_span(start, line, col),
565                "hex color must be #RRGGBB or #RRGGBBAA",
566            ))
567        }
568    }
569
570    fn read_number(&mut self, first: char) -> Token {
571        let (start, line, col) = self.span_start();
572        self.advance(); // consume the first char (caller only peeked)
573        let mut s = String::from(first);
574        while let Some(c) = self.peek() {
575            if c.is_ascii_digit() || c == '.' {
576                s.push(c);
577                self.advance();
578            } else {
579                break;
580            }
581        }
582        let n: f64 = s.parse().unwrap_or(0.0);
583        Token {
584            kind: TokenKind::Number(n),
585            span: self.make_span(start, line, col),
586        }
587    }
588
589    fn read_ident_or_keyword(&mut self, first: char) -> Token {
590        let (start, line, col) = self.span_start();
591        self.advance(); // consume the first char (caller only peeked)
592        let mut s = String::from(first);
593        while let Some(c) = self.peek() {
594            if c.is_ascii_alphanumeric() || c == '_' {
595                s.push(c);
596                self.advance();
597            } else {
598                break;
599            }
600        }
601        let kind = match s.as_str() {
602            "true" => TokenKind::True,
603            "false" => TokenKind::False,
604            "let" => TokenKind::Let,
605            "state" => TokenKind::State,
606            "component" => TokenKind::Component,
607            "screen" => TokenKind::Screen,
608            "if" => TokenKind::If,
609            "else" => TokenKind::Else,
610            "for" => TokenKind::For,
611            "in" => TokenKind::In,
612            "theme" => TokenKind::Theme,
613            "use" => TokenKind::Use,
614            "import" => TokenKind::Import,
615            "header" => TokenKind::Header,
616            "footer" => TokenKind::Footer,
617            "container" => TokenKind::Container,
618            "sidebar" => TokenKind::Sidebar,
619            "box" => TokenKind::Box,
620            "text" => TokenKind::Text,
621            "row" => TokenKind::Row,
622            "column" => TokenKind::Column,
623            "grid" => TokenKind::Grid,
624            "stack" => TokenKind::Stack,
625            "center" => TokenKind::Center,
626            "spacer" => TokenKind::Spacer,
627            "image" => TokenKind::Image,
628            "button" => TokenKind::Button,
629            "input" => TokenKind::Input,
630            "card" => TokenKind::Card,
631            "section" => TokenKind::Section,
632            "widget" => TokenKind::Widget,
633            "accordion" => TokenKind::Accordion,
634            "bento" => TokenKind::Bento,
635            "breadcrumb" => TokenKind::Breadcrumb,
636            "hamburger" => TokenKind::Hamburger,
637            "kebab" => TokenKind::Kebab,
638            "meatballs" => TokenKind::Meatballs,
639            "doner" => TokenKind::Doner,
640            "tabs" => TokenKind::Tabs,
641            "pagination" => TokenKind::Pagination,
642            "linkList" => TokenKind::LinkList,
643            "nav" => TokenKind::Nav,
644            "password" => TokenKind::Password,
645            "search" => TokenKind::Search,
646            "checkbox" => TokenKind::Checkbox,
647            "radio" => TokenKind::Radio,
648            "dropdown" => TokenKind::Dropdown,
649            "combobox" => TokenKind::Combobox,
650            "multiselect" => TokenKind::Multiselect,
651            "datePicker" => TokenKind::DatePicker,
652            "picker" => TokenKind::Picker,
653            "slider" => TokenKind::Slider,
654            "stepper" => TokenKind::Stepper,
655            "toggle" => TokenKind::Toggle,
656            "form" => TokenKind::Form,
657            "modal" => TokenKind::Modal,
658            "confirmDialog" => TokenKind::ConfirmDialog,
659            "toast" => TokenKind::Toast,
660            "notification" => TokenKind::Notification,
661            "alert" => TokenKind::Alert,
662            "messageBox" => TokenKind::MessageBox,
663            "tooltip" => TokenKind::Tooltip,
664            "loader" => TokenKind::Loader,
665            "progressBar" => TokenKind::ProgressBar,
666            "badge" => TokenKind::Badge,
667            "icon" => TokenKind::Icon,
668            "tag" => TokenKind::Tag,
669            "comment" => TokenKind::Comment,
670            "feed" => TokenKind::Feed,
671            "carousel" => TokenKind::Carousel,
672            "chart" => TokenKind::Chart,
673            "table" => TokenKind::Table,
674            "avatar" => TokenKind::Avatar,
675            "skeleton" => TokenKind::Skeleton,
676            "drawer" => TokenKind::Drawer,
677            "select" => TokenKind::Select,
678            "textarea" => TokenKind::Textarea,
679            "popover" => TokenKind::Popover,
680            "separator" => TokenKind::Separator,
681            "timeline" => TokenKind::Timeline,
682            "rating" => TokenKind::Rating,
683            "fileUpload" => TokenKind::FileUpload,
684            "colorPicker" => TokenKind::ColorPicker,
685            "treeView" => TokenKind::TreeView,
686            "commandPalette" => TokenKind::CommandPalette,
687            "splitter" => TokenKind::Splitter,
688            "width" | "height" | "fill" | "stroke" | "radius" | "padding" | "gap"
689            | "grow" | "shrink" | "align" | "justify" | "direction"
690            | "fontSize" | "fontWeight" | "shadow" => {
691                // Treat as identifier so "let padding = 24" works; parser uses Ident for prop names
692                TokenKind::Ident(s)
693            }
694            _ => TokenKind::Ident(s),
695        };
696        Token {
697            kind,
698            span: self.make_span(start, line, col),
699        }
700    }
701
702    pub fn next_token(&mut self) -> Result<Token, NewtError> {
703        self.skip_whitespace_and_comments();
704        let (start, line, col) = self.span_start();
705        let c = match self.peek() {
706            Some(c) => c,
707            None => {
708                return Ok(Token {
709                    kind: TokenKind::Eof,
710                    span: self.make_span(start, line, col),
711                });
712            }
713        };
714
715        // Two-char
716        let (a, b) = self.peek2();
717        if a == Some('-') && b == Some('>') {
718            self.advance();
719            self.advance();
720            return Ok(Token {
721                kind: TokenKind::Arrow,
722                span: self.make_span(start, line, col),
723            });
724        }
725        if a == Some('=') && b == Some('=') {
726            self.advance();
727            self.advance();
728            return Ok(Token {
729                kind: TokenKind::EqEq,
730                span: self.make_span(start, line, col),
731            });
732        }
733        if a == Some('!') && b == Some('=') {
734            self.advance();
735            self.advance();
736            return Ok(Token {
737                kind: TokenKind::NotEq,
738                span: self.make_span(start, line, col),
739            });
740        }
741        if a == Some('<') && b == Some('=') {
742            self.advance();
743            self.advance();
744            return Ok(Token {
745                kind: TokenKind::Le,
746                span: self.make_span(start, line, col),
747            });
748        }
749        if a == Some('>') && b == Some('=') {
750            self.advance();
751            self.advance();
752            return Ok(Token {
753                kind: TokenKind::Ge,
754                span: self.make_span(start, line, col),
755            });
756        }
757        if a == Some('&') && b == Some('&') {
758            self.advance();
759            self.advance();
760            return Ok(Token {
761                kind: TokenKind::And,
762                span: self.make_span(start, line, col),
763            });
764        }
765        if a == Some('|') && b == Some('|') {
766            self.advance();
767            self.advance();
768            return Ok(Token {
769                kind: TokenKind::Or,
770                span: self.make_span(start, line, col),
771            });
772        }
773
774        // Single char
775        let kind = match c {
776            '"' => return self.read_string(),
777            '{' => TokenKind::LeftBrace,
778            '}' => TokenKind::RightBrace,
779            '(' => TokenKind::LeftParen,
780            ')' => TokenKind::RightParen,
781            '[' => TokenKind::LeftBracket,
782            ']' => TokenKind::RightBracket,
783            ',' => TokenKind::Comma,
784            '.' => TokenKind::Dot,
785            ';' => TokenKind::Semicolon,
786            ':' => TokenKind::Colon,
787            '=' => TokenKind::Eq,
788            '+' => TokenKind::Plus,
789            '-' => TokenKind::Minus,
790            '*' => TokenKind::Star,
791            '/' => TokenKind::Slash,
792            '%' => TokenKind::Percent,
793            '<' => TokenKind::Lt,
794            '>' => TokenKind::Gt,
795            '!' => TokenKind::Not,
796            _ if c.is_ascii_digit() => {
797                let t = self.read_number(c);
798                return Ok(t);
799            }
800            _ if c == '#' => {
801                let t = self.read_hex_color()?;
802                return Ok(t);
803            }
804            _ if c.is_ascii_alphabetic() || c == '_' => {
805                let t = self.read_ident_or_keyword(c);
806                return Ok(t);
807            }
808            _ => {
809                return Err(NewtError::lexer(
810                    self.make_span(start, line, col),
811                    format!("unexpected character '{}'", c),
812                ));
813            }
814        };
815        self.advance();
816        Ok(Token {
817            kind,
818            span: self.make_span(start, line, col),
819        })
820    }
821}
newter_compiler/lexer.rs

newter_compiler/
lexer.rs
