ternlang_core/
lexer.rs

1use logos::Logos;
2
3#[derive(Logos, Debug, PartialEq, Clone)]
4#[logos(skip r"[ \t\n\f]+")] // Skip whitespace
5#[logos(skip(r"//[^\n]*", allow_greedy = true))]   // Skip line comments
6#[logos(skip(r"/\*[^*]*\*+(?:[^*/][^*]*\*+)*/", allow_greedy = true))]  // Skip block comments /* ... */
7pub enum Token {
8    #[regex(r"[0-9]+\.[0-9]+", |lex| lex.slice().parse::<f64>().ok(), priority = 100)]
9    Float(f64),
10
11    // Ternary Specific
12    #[token("affirm", priority = 4)]
13    Affirm,
14
15    #[token("tend", priority = 4)]
16    Tend,
17
18    #[token("reject", priority = 4)]
19    Reject,
20
21    #[token("?")]
22    UncertainBranch,
23
24    #[token("trit", priority = 3)]
25    TritType,
26
27    #[token("trittensor", priority = 3)]
28    TritTensor,
29
30    #[token("sparseskip", priority = 3)]
31    SparseSkip,
32
33
34    // Standard Keywords
35    #[token("if", priority = 3)]
36    If,
37
38    #[token("else", priority = 3)]
39    Else,
40
41    #[token("fn", priority = 3)]
42    Fn,
43
44    #[token("return", priority = 3)]
45    Return,
46
47    #[token("let", priority = 3)]
48    Let,
49
50    #[token("match", priority = 3)]
51    Match,
52
53    #[token("for", priority = 3)]
54    For,
55
56    #[token("in", priority = 3)]
57    In,
58
59    #[token("while", priority = 3)]
60    While,
61
62    #[token("loop", priority = 3)]
63    Loop,
64
65    #[token("break", priority = 3)]
66    Break,
67
68    #[token("continue", priority = 3)]
69    Continue,
70
71    #[token("mut", priority = 3)]
72    Mut,
73
74    #[token("use", priority = 3)]
75    Use,
76
77    #[token("from", priority = 3)]
78    From,
79
80    #[token("import", priority = 3)]
81    Import,
82
83    #[token("module", priority = 3)]
84    Module,
85
86    #[token("pub", priority = 3)]
87    Pub,
88
89    #[token("struct", priority = 3)]
90    Struct,
91
92    // Actor model keywords
93    #[token("agent", priority = 3)]
94    Agent,
95
96    #[token("spawn", priority = 3)]
97    Spawn,
98
99    #[token("send", priority = 3)]
100    Send,
101
102    #[token("await", priority = 3)]
103    Await,
104
105    #[token("agentref", priority = 3)]
106    AgentRef,
107
108    #[token("remote", priority = 3)]
109    Remote,
110
111    #[token("nodeid", priority = 3)]
112    NodeId,
113
114    // Operators
115    #[token("+")]
116    Plus,
117
118    #[token("-")]
119    Minus,
120
121    #[token("*")]
122    Star,
123
124    #[token("/")]
125    Slash,
126
127    #[token("%")]
128    Percent,
129
130    #[token("=")]
131    Assign,
132
133    #[token("==")]
134    Equal,
135
136    #[token("=>")]
137    FatArrow,
138
139    #[token("!=")]
140    NotEqual,
141
142    #[token("::")]
143    DoubleColon,
144
145    #[token("&&")]
146    And,
147
148    #[token("||")]
149    Or,
150
151    // Delimiters
152    #[token("(", priority = 3)]
153    LParen,
154
155    #[token(")", priority = 3)]
156    RParen,
157
158    #[token("{", priority = 3)]
159    LBrace,
160
161    #[token("}", priority = 3)]
162    RBrace,
163
164    #[token("[", priority = 3)]
165    LBracket,
166
167    #[token("]", priority = 3)]
168    RBracket,
169
170    #[token("<", priority = 3)]
171    LAngle,
172
173    #[token(">", priority = 3)]
174    RAngle,
175
176    #[token("<=", priority = 4)]
177    LessEqual,
178
179    #[token(">=", priority = 4)]
180    GreaterEqual,
181
182    #[token(",", priority = 3)]
183    Comma,
184
185    #[token(";", priority = 3)]
186    Semicolon,
187
188    #[token(":")]
189    Colon,
190
191    #[token("@")]
192    At,
193
194    #[token("->")]
195    Arrow,
196
197    #[token(".")]
198    Dot,
199
200    // Literals
201    #[regex("[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
202    Ident(String),
203
204    // PARSER-LIT-001: hex (0xFF) and binary (0b1010) integer literals.
205    // Must have higher priority than the plain decimal rule so the '0' prefix is
206    // consumed as part of the full literal rather than tokenised as Int(0) + Ident.
207    #[regex(r"0[xX][0-9a-fA-F]+", |lex| i64::from_str_radix(&lex.slice()[2..], 16).ok(), priority = 20)]
208    #[regex(r"0[bB][01]+", |lex| i64::from_str_radix(&lex.slice()[2..], 2).ok(), priority = 20)]
209    #[regex("[0-9]+", |lex| lex.slice().parse::<i64>().ok(), priority = 10)]
210    Int(i64),
211
212    /// Double-quoted string literal: "value"
213    #[regex(r#""[^"]*""#, |lex| {
214        let s = lex.slice();
215        Some(s[1..s.len()-1].to_string())
216    }, priority = 2)]
217    StringLit(String),
218
219    // Unused but kept for variant compatibility
220    #[token("___TRIT_LITERAL_UNUSED___")]
221    TritLiteral,
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227
228    #[test]
229    fn test_lexer() {
230        let input = "let x: trit = 1; if x ? { return 0; }";
231        let mut lex = Token::lexer(input);
232
233        assert_eq!(lex.next(), Some(Ok(Token::Let)));
234        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
235        assert_eq!(lex.next(), Some(Ok(Token::Colon)));
236        assert_eq!(lex.next(), Some(Ok(Token::TritType)));
237        assert_eq!(lex.next(), Some(Ok(Token::Assign)));
238        assert_eq!(lex.next(), Some(Ok(Token::Int(1))));
239        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
240        assert_eq!(lex.next(), Some(Ok(Token::If)));
241        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
242        assert_eq!(lex.next(), Some(Ok(Token::UncertainBranch)));
243        assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
244        assert_eq!(lex.next(), Some(Ok(Token::Return)));
245        assert_eq!(lex.next(), Some(Ok(Token::Int(0))));
246        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
247        assert_eq!(lex.next(), Some(Ok(Token::RBrace)));
248    }
249}
ternlang_core/lexer.rs

ternlang_core/
lexer.rs
