glyph_parser/
lexer.rs

1//! Lexical analyzer for Glyph
2
3use logos::Logos;
4
5#[derive(Logos, Debug, PartialEq, Clone)]
6#[logos(skip r"[ \t]+")]
7#[logos(skip r"#[^\n]*")]
8pub enum Token {
9    // Keywords
10    #[token("and")]
11    And,
12    #[token("as")]
13    As,
14    #[token("assert")]
15    Assert,
16    #[token("async")]
17    Async,
18    #[token("await")]
19    Await,
20    #[token("break")]
21    Break,
22    #[token("class")]
23    Class,
24    #[token("continue")]
25    Continue,
26    #[token("def")]
27    Def,
28    #[token("del")]
29    Del,
30    #[token("elif")]
31    Elif,
32    #[token("else")]
33    Else,
34    #[token("except")]
35    Except,
36    #[token("False")]
37    False,
38    #[token("finally")]
39    Finally,
40    #[token("for")]
41    For,
42    #[token("from")]
43    From,
44    #[token("global")]
45    Global,
46    #[token("if")]
47    If,
48    #[token("import")]
49    Import,
50    #[token("in")]
51    In,
52    #[token("is")]
53    Is,
54    #[token("lambda")]
55    Lambda,
56    #[token("let")]
57    Let,
58    #[token("match")]
59    Match,
60    #[token("case")]
61    Case,
62    #[token("None")]
63    None,
64    #[token("nonlocal")]
65    Nonlocal,
66    #[token("not")]
67    Not,
68    #[token("or")]
69    Or,
70    #[token("pass")]
71    Pass,
72    #[token("raise")]
73    Raise,
74    #[token("return")]
75    Return,
76    #[token("True")]
77    True,
78    #[token("try")]
79    Try,
80    #[token("while")]
81    While,
82    #[token("with")]
83    With,
84    #[token("yield")]
85    Yield,
86
87    // Literals (simplified - no data attached)
88    #[regex(r"-?[0-9]+")]
89    Int,
90
91    #[regex(r"-?[0-9]+\.[0-9]+")]
92    Float,
93
94    #[regex(r#""([^"\\]|\\.)*""#)]
95    #[regex(r#"'([^'\\]|\\.)*'"#)]
96    String,
97
98    #[regex(r#"f"([^"\\]|\\.)*""#)]
99    #[regex(r#"f'([^'\\]|\\.)*'"#)]
100    FString,
101
102    // Identifiers
103    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*")]
104    Identifier,
105
106    // Operators
107    #[token("+")]
108    Plus,
109    #[token("-")]
110    Minus,
111    #[token("*")]
112    Star,
113    #[token("**")]
114    StarStar,
115    #[token("/")]
116    Slash,
117    #[token("//")]
118    SlashSlash,
119    #[token("%")]
120    Percent,
121    #[token("@")]
122    At,
123    #[token("<<")]
124    LeftShift,
125    #[token(">>")]
126    RightShift,
127    #[token("&")]
128    Ampersand,
129    #[token("|")]
130    Pipe,
131    #[token("^")]
132    Caret,
133    #[token("~")]
134    Tilde,
135    #[token("<")]
136    Less,
137    #[token(">")]
138    Greater,
139    #[token("<=")]
140    LessEqual,
141    #[token(">=")]
142    GreaterEqual,
143    #[token("==")]
144    EqualEqual,
145    #[token("!=")]
146    NotEqual,
147
148    // Delimiters
149    #[token("(")]
150    LeftParen,
151    #[token(")")]
152    RightParen,
153    #[token("[")]
154    LeftBracket,
155    #[token("]")]
156    RightBracket,
157    #[token("{")]
158    LeftBrace,
159    #[token("}")]
160    RightBrace,
161    #[token(",")]
162    Comma,
163    #[token(":")]
164    Colon,
165    #[token(".")]
166    Dot,
167    #[token(";")]
168    Semicolon,
169    #[token("=")]
170    Equal,
171    #[token("->")]
172    Arrow,
173
174    // Special
175    #[token("\n")]
176    Newline,
177
178    // Indentation tokens (handled separately)
179    Indent,
180    Dedent,
181}
182
183impl Token {
184    pub fn is_trivia(&self) -> bool {
185        false // We're skipping whitespace and comments via logos
186    }
187}
glyph_parser/lexer.rs

glyph_parser/
lexer.rs
