sylt_tokenizer/
token.rs

1use logos::Logos;
2
3#[derive(Logos, Debug, PartialEq, Clone)]
4pub enum Token {
5    #[regex(r"[A-Za-z_][A-Za-z0-9_]*", |lex| lex.slice().to_string())]
6    Identifier(String),
7
8    #[regex(r#""[^"]*""#, |lex| { let mut s = lex.slice().to_string(); s.remove(0); s.pop(); s })]
9    String(String),
10
11    #[regex(r"[\d]+\.[\d]*|[\d]*\.[\d]+", |lex| lex.slice().parse(), priority=2)]
12    Float(f64),
13    #[regex(r"[\d]+", |lex| lex.slice().parse())]
14    Int(i64),
15
16    #[regex(r"nil")]
17    Nil,
18
19    #[regex(r"true|false", |lex| lex.slice().parse(), priority=2)]
20    Bool(bool),
21
22    #[token("if")]
23    If,
24    #[token("is")]
25    Is,
26    #[token("else")]
27    Else,
28    #[token("break")]
29    Break,
30    #[token("continue")]
31    Continue,
32    #[token("in")]
33    In,
34    #[token("loop")]
35    Loop,
36    #[token("blob")]
37    Blob,
38
39    #[token("print")]
40    Print,
41
42    #[token("yield")]
43    Yield,
44
45    #[token("ret")]
46    Ret,
47
48    #[token("+")]
49    Plus,
50    #[token("++")]
51    PlusPlus,
52    #[token("-")]
53    Minus,
54    #[token("--")]
55    MinusMinus,
56    #[token("*")]
57    Star,
58    #[token("/")]
59    Slash,
60    #[token("+=")]
61    PlusEqual,
62    #[token("-=")]
63    MinusEqual,
64    #[token("*=")]
65    StarEqual,
66    #[token("/=")]
67    SlashEqual,
68
69    #[token("#")]
70    Hash,
71    #[token(":")]
72    Colon,
73    #[token("::")]
74    ColonColon,
75    #[token(":=")]
76    ColonEqual,
77    #[token("=")]
78    Equal,
79    #[token("==")]
80    EqualEqual,
81    #[token("!=")]
82    NotEqual,
83
84    #[token("<=>")]
85    AssertEqual,
86    #[token("<!>")]
87    Unreachable,
88
89    #[token("(")]
90    LeftParen,
91    #[token(")")]
92    RightParen,
93
94    #[token("[")]
95    LeftBracket,
96    #[token("]")]
97    RightBracket,
98
99    #[token("{")]
100    LeftBrace,
101    #[token("}")]
102    RightBrace,
103
104    #[token(">")]
105    Greater,
106    #[token(">=")]
107    GreaterEqual,
108    #[token("<")]
109    Less,
110    #[token("<=")]
111    LessEqual,
112
113    #[token("fn")]
114    Fn,
115
116    #[token("&&")]
117    And,
118    #[token("||")]
119    Or,
120    #[token("!")]
121    Bang,
122    #[token("?")]
123    QuestionMark,
124    #[token("|")]
125    Pipe,
126    #[token("'")]
127    Prime,
128
129    #[token(",")]
130    Comma,
131    #[token(".")]
132    Dot,
133    #[token("->")]
134    Arrow,
135    #[token("\n")]
136    Newline,
137
138    #[token("use")]
139    Use,
140
141    #[token("<<<<<<<")]
142    GitConflictBegin,
143    #[token(">>>>>>>")]
144    GitConflictEnd,
145
146    #[regex(r"//[^\n]*", logos::skip)]
147    Comment,
148
149    #[regex(r"[ \t\r]", logos::skip)]
150    Whitespace,
151
152    EOF,
153
154    #[error]
155    Error,
156}
sylt_tokenizer/token.rs

sylt_tokenizer/
token.rs
