ternlang_core/
lexer.rs

1use logos::Logos;
2
3#[derive(Logos, Debug, PartialEq, Clone)]
4#[logos(skip r"[ \t\n\f]+")] // Skip whitespace
5#[logos(skip(r"//[^\n]*", allow_greedy = true))]   // Skip line comments
6#[logos(skip(r"/\*[^*]*\*+(?:[^*/][^*]*\*+)*/", allow_greedy = true))]  // Skip block comments /* ... */
7pub enum Token {
8    #[regex(r"[0-9]+\.[0-9]+", |lex| lex.slice().parse::<f64>().ok(), priority = 100)]
9    Float(f64),
10
11    // Ternary Specific
12    #[token("affirm", priority = 4)]
13    Affirm,
14
15    #[token("tend", priority = 4)]
16    Tend,
17
18    #[token("reject", priority = 4)]
19    Reject,
20
21    #[token("?")]
22    UncertainBranch,
23
24    #[token("trit", priority = 3)]
25    TritType,
26
27    #[token("trittensor", priority = 3)]
28    TritTensor,
29
30    #[token("sparseskip", priority = 3)]
31    SparseSkip,
32
33
34    // Standard Keywords
35    #[token("if", priority = 3)]
36    If,
37
38    #[token("else", priority = 3)]
39    Else,
40
41    #[token("fn", priority = 3)]
42    Fn,
43
44    #[token("return", priority = 3)]
45    Return,
46
47    #[token("let", priority = 3)]
48    Let,
49
50    #[token("match", priority = 3)]
51    Match,
52
53    #[token("for", priority = 3)]
54    For,
55
56    #[token("in", priority = 3)]
57    In,
58
59    #[token("while", priority = 3)]
60    While,
61
62    #[token("loop", priority = 3)]
63    Loop,
64
65    #[token("break", priority = 3)]
66    Break,
67
68    #[token("continue", priority = 3)]
69    Continue,
70
71    #[token("mut", priority = 3)]
72    Mut,
73
74    #[token("use", priority = 3)]
75    Use,
76
77    #[token("module", priority = 3)]
78    Module,
79
80    #[token("pub", priority = 3)]
81    Pub,
82
83    #[token("struct", priority = 3)]
84    Struct,
85
86    // Actor model keywords
87    #[token("agent", priority = 3)]
88    Agent,
89
90    #[token("spawn", priority = 3)]
91    Spawn,
92
93    #[token("send", priority = 3)]
94    Send,
95
96    #[token("await", priority = 3)]
97    Await,
98
99    #[token("agentref", priority = 3)]
100    AgentRef,
101
102    #[token("remote", priority = 3)]
103    Remote,
104
105    #[token("nodeid", priority = 3)]
106    NodeId,
107
108    // Operators
109    #[token("+")]
110    Plus,
111
112    #[token("-")]
113    Minus,
114
115    #[token("*")]
116    Star,
117
118    #[token("/")]
119    Slash,
120
121    #[token("%")]
122    Percent,
123
124    #[token("=")]
125    Assign,
126
127    #[token("==")]
128    Equal,
129
130    #[token("=>")]
131    FatArrow,
132
133    #[token("!=")]
134    NotEqual,
135
136    #[token("::")]
137    DoubleColon,
138
139    #[token("&&")]
140    And,
141
142    #[token("||")]
143    Or,
144
145    // Delimiters
146    #[token("(", priority = 3)]
147    LParen,
148
149    #[token(")", priority = 3)]
150    RParen,
151
152    #[token("{", priority = 3)]
153    LBrace,
154
155    #[token("}", priority = 3)]
156    RBrace,
157
158    #[token("[", priority = 3)]
159    LBracket,
160
161    #[token("]", priority = 3)]
162    RBracket,
163
164    #[token("<", priority = 3)]
165    LAngle,
166
167    #[token(">", priority = 3)]
168    RAngle,
169
170    #[token("<=", priority = 4)]
171    LessEqual,
172
173    #[token(">=", priority = 4)]
174    GreaterEqual,
175
176    #[token(",", priority = 3)]
177    Comma,
178
179    #[token(";", priority = 3)]
180    Semicolon,
181
182    #[token(":")]
183    Colon,
184
185    #[token("@")]
186    At,
187
188    #[token("->")]
189    Arrow,
190
191    #[token(".")]
192    Dot,
193
194    // Literals
195    #[regex("[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
196    Ident(String),
197
198    #[regex("[0-9]+", |lex| lex.slice().parse::<i64>().ok(), priority = 10)]
199    Int(i64),
200
201    /// Double-quoted string literal: "value"
202    #[regex(r#""[^"]*""#, |lex| {
203        let s = lex.slice();
204        Some(s[1..s.len()-1].to_string())
205    }, priority = 2)]
206    StringLit(String),
207
208    // Unused but kept for variant compatibility
209    #[token("___TRIT_LITERAL_UNUSED___")]
210    TritLiteral,
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216
217    #[test]
218    fn test_lexer() {
219        let input = "let x: trit = 1; if x ? { return 0; }";
220        let mut lex = Token::lexer(input);
221
222        assert_eq!(lex.next(), Some(Ok(Token::Let)));
223        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
224        assert_eq!(lex.next(), Some(Ok(Token::Colon)));
225        assert_eq!(lex.next(), Some(Ok(Token::TritType)));
226        assert_eq!(lex.next(), Some(Ok(Token::Assign)));
227        assert_eq!(lex.next(), Some(Ok(Token::Int(1))));
228        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
229        assert_eq!(lex.next(), Some(Ok(Token::If)));
230        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
231        assert_eq!(lex.next(), Some(Ok(Token::UncertainBranch)));
232        assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
233        assert_eq!(lex.next(), Some(Ok(Token::Return)));
234        assert_eq!(lex.next(), Some(Ok(Token::Int(0))));
235        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
236        assert_eq!(lex.next(), Some(Ok(Token::RBrace)));
237    }
238}
ternlang_core/lexer.rs

ternlang_core/
lexer.rs
