ternlang_core/
lexer.rs

1use logos::Logos;
2
3#[derive(Logos, Debug, PartialEq, Clone)]
4#[logos(skip r"[ \t\n\f]+")] // Skip whitespace
5#[logos(skip(r"//[^\n]*", allow_greedy = true))]   // Skip line comments
6pub enum Token {
7    // Ternary Specific
8    #[token("-1")]
9    #[token("0")]
10    #[token("1")]
11    TritLiteral,
12
13    #[token("?")]
14    UncertainBranch,
15
16    #[token("trit", priority = 3)]
17    TritType,
18
19    #[token("trittensor", priority = 3)]
20    TritTensor,
21
22    #[token("sparseskip", priority = 3)]
23    SparseSkip,
24
25
26    // Standard Keywords
27    #[token("if", priority = 3)]
28    If,
29
30    #[token("else", priority = 3)]
31    Else,
32
33    #[token("fn", priority = 3)]
34    Fn,
35
36    #[token("return", priority = 3)]
37    Return,
38
39    #[token("let", priority = 3)]
40    Let,
41
42    #[token("match", priority = 3)]
43    Match,
44
45    #[token("for", priority = 3)]
46    For,
47
48    #[token("in", priority = 3)]
49    In,
50
51    #[token("while", priority = 3)]
52    While,
53
54    #[token("loop", priority = 3)]
55    Loop,
56
57    #[token("break", priority = 3)]
58    Break,
59
60    #[token("continue", priority = 3)]
61    Continue,
62
63    #[token("mut", priority = 3)]
64    Mut,
65
66    #[token("use", priority = 3)]
67    Use,
68
69    #[token("module", priority = 3)]
70    Module,
71
72    #[token("pub", priority = 3)]
73    Pub,
74
75    #[token("struct", priority = 3)]
76    Struct,
77
78    // Actor model keywords
79    #[token("agent", priority = 3)]
80    Agent,
81
82    #[token("spawn", priority = 3)]
83    Spawn,
84
85    #[token("send", priority = 3)]
86    Send,
87
88    #[token("await", priority = 3)]
89    Await,
90
91    #[token("agentref", priority = 3)]
92    AgentRef,
93
94    #[token("remote", priority = 3)]
95    Remote,
96
97    #[token("nodeid", priority = 3)]
98    NodeId,
99
100    // Operators
101    #[token("+")]
102    Plus,
103
104    #[token("-")]
105    Minus,
106
107    #[token("*")]
108    Star,
109
110    #[token("=")]
111    Assign,
112
113    #[token("==")]
114    Equal,
115
116    #[token("=>")]
117    FatArrow,
118
119    #[token("!=")]
120    NotEqual,
121
122    #[token("::")]
123    DoubleColon,
124
125    #[token("&&")]
126    And,
127
128    #[token("||")]
129    Or,
130
131    // Delimiters
132    #[token("(", priority = 3)]
133    LParen,
134
135    #[token(")", priority = 3)]
136    RParen,
137
138    #[token("{", priority = 3)]
139    LBrace,
140
141    #[token("}", priority = 3)]
142    RBrace,
143
144    #[token("[", priority = 3)]
145    LBracket,
146
147    #[token("]", priority = 3)]
148    RBracket,
149
150    #[token("<", priority = 3)]
151    LAngle,
152
153    #[token(">", priority = 3)]
154    RAngle,
155
156    #[token(",", priority = 3)]
157    Comma,
158
159    #[token(";", priority = 3)]
160    Semicolon,
161
162    #[token(":")]
163    Colon,
164
165    #[token("@")]
166    At,
167
168    #[token("->")]
169    Arrow,
170
171    #[token(".")]
172    Dot,
173
174    // Literals
175    #[regex("[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
176    Ident(String),
177
178    #[regex("[0-9]+", |lex| lex.slice().parse::<i64>().ok(), priority = 1)]
179    Int(i64),
180
181    /// Double-quoted string literal: "value"
182    #[regex(r#""[^"]*""#, |lex| {
183        let s = lex.slice();
184        Some(s[1..s.len()-1].to_string())
185    }, priority = 2)]
186    StringLit(String),
187}
188
189#[cfg(test)]
190mod tests {
191    use super::*;
192
193    #[test]
194    fn test_lexer() {
195        let input = "let x: trit = 1; if x ? { return 0; }";
196        let mut lex = Token::lexer(input);
197
198        assert_eq!(lex.next(), Some(Ok(Token::Let)));
199        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
200        assert_eq!(lex.next(), Some(Ok(Token::Colon)));
201        assert_eq!(lex.next(), Some(Ok(Token::TritType)));
202        assert_eq!(lex.next(), Some(Ok(Token::Assign)));
203        assert_eq!(lex.next(), Some(Ok(Token::TritLiteral)));
204        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
205        assert_eq!(lex.next(), Some(Ok(Token::If)));
206        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
207        assert_eq!(lex.next(), Some(Ok(Token::UncertainBranch)));
208        assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
209        assert_eq!(lex.next(), Some(Ok(Token::Return)));
210        assert_eq!(lex.next(), Some(Ok(Token::TritLiteral)));
211        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
212        assert_eq!(lex.next(), Some(Ok(Token::RBrace)));
213    }
214}
ternlang_core/lexer.rs

ternlang_core/
lexer.rs
