ternlang_core/
lexer.rs

1use logos::Logos;
2
3#[derive(Logos, Debug, PartialEq, Clone)]
4#[logos(skip r"[ \t\n\f]+")] // Skip whitespace
5#[logos(skip(r"//[^\n]*", allow_greedy = true))]   // Skip line comments
6pub enum Token {
7    // Ternary Specific
8    #[token("-1")]
9    #[token("0")]
10    #[token("1")]
11    TritLiteral,
12
13    // Ternary semantic keywords — first-class trit values
14    #[token("affirm", priority = 4)]
15    Affirm,
16
17    #[token("tend", priority = 4)]
18    Tend,
19
20    #[token("reject", priority = 4)]
21    Reject,
22
23    #[token("?")]
24    UncertainBranch,
25
26    #[token("trit", priority = 3)]
27    TritType,
28
29    #[token("trittensor", priority = 3)]
30    TritTensor,
31
32    #[token("sparseskip", priority = 3)]
33    SparseSkip,
34
35
36    // Standard Keywords
37    #[token("if", priority = 3)]
38    If,
39
40    #[token("else", priority = 3)]
41    Else,
42
43    #[token("fn", priority = 3)]
44    Fn,
45
46    #[token("return", priority = 3)]
47    Return,
48
49    #[token("let", priority = 3)]
50    Let,
51
52    #[token("match", priority = 3)]
53    Match,
54
55    #[token("for", priority = 3)]
56    For,
57
58    #[token("in", priority = 3)]
59    In,
60
61    #[token("while", priority = 3)]
62    While,
63
64    #[token("loop", priority = 3)]
65    Loop,
66
67    #[token("break", priority = 3)]
68    Break,
69
70    #[token("continue", priority = 3)]
71    Continue,
72
73    #[token("mut", priority = 3)]
74    Mut,
75
76    #[token("use", priority = 3)]
77    Use,
78
79    #[token("module", priority = 3)]
80    Module,
81
82    #[token("pub", priority = 3)]
83    Pub,
84
85    #[token("struct", priority = 3)]
86    Struct,
87
88    // Actor model keywords
89    #[token("agent", priority = 3)]
90    Agent,
91
92    #[token("spawn", priority = 3)]
93    Spawn,
94
95    #[token("send", priority = 3)]
96    Send,
97
98    #[token("await", priority = 3)]
99    Await,
100
101    #[token("agentref", priority = 3)]
102    AgentRef,
103
104    #[token("remote", priority = 3)]
105    Remote,
106
107    #[token("nodeid", priority = 3)]
108    NodeId,
109
110    // Operators
111    #[token("+")]
112    Plus,
113
114    #[token("-")]
115    Minus,
116
117    #[token("*")]
118    Star,
119
120    #[token("=")]
121    Assign,
122
123    #[token("==")]
124    Equal,
125
126    #[token("=>")]
127    FatArrow,
128
129    #[token("!=")]
130    NotEqual,
131
132    #[token("::")]
133    DoubleColon,
134
135    #[token("&&")]
136    And,
137
138    #[token("||")]
139    Or,
140
141    // Delimiters
142    #[token("(", priority = 3)]
143    LParen,
144
145    #[token(")", priority = 3)]
146    RParen,
147
148    #[token("{", priority = 3)]
149    LBrace,
150
151    #[token("}", priority = 3)]
152    RBrace,
153
154    #[token("[", priority = 3)]
155    LBracket,
156
157    #[token("]", priority = 3)]
158    RBracket,
159
160    #[token("<", priority = 3)]
161    LAngle,
162
163    #[token(">", priority = 3)]
164    RAngle,
165
166    #[token(",", priority = 3)]
167    Comma,
168
169    #[token(";", priority = 3)]
170    Semicolon,
171
172    #[token(":")]
173    Colon,
174
175    #[token("@")]
176    At,
177
178    #[token("->")]
179    Arrow,
180
181    #[token(".")]
182    Dot,
183
184    // Literals
185    #[regex("[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string(), priority = 1)]
186    Ident(String),
187
188    #[regex("[0-9]+", |lex| lex.slice().parse::<i64>().ok(), priority = 1)]
189    Int(i64),
190
191    /// Double-quoted string literal: "value"
192    #[regex(r#""[^"]*""#, |lex| {
193        let s = lex.slice();
194        Some(s[1..s.len()-1].to_string())
195    }, priority = 2)]
196    StringLit(String),
197}
198
199#[cfg(test)]
200mod tests {
201    use super::*;
202
203    #[test]
204    fn test_lexer() {
205        let input = "let x: trit = 1; if x ? { return 0; }";
206        let mut lex = Token::lexer(input);
207
208        assert_eq!(lex.next(), Some(Ok(Token::Let)));
209        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
210        assert_eq!(lex.next(), Some(Ok(Token::Colon)));
211        assert_eq!(lex.next(), Some(Ok(Token::TritType)));
212        assert_eq!(lex.next(), Some(Ok(Token::Assign)));
213        assert_eq!(lex.next(), Some(Ok(Token::TritLiteral)));
214        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
215        assert_eq!(lex.next(), Some(Ok(Token::If)));
216        assert_eq!(lex.next(), Some(Ok(Token::Ident("x".to_string()))));
217        assert_eq!(lex.next(), Some(Ok(Token::UncertainBranch)));
218        assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
219        assert_eq!(lex.next(), Some(Ok(Token::Return)));
220        assert_eq!(lex.next(), Some(Ok(Token::TritLiteral)));
221        assert_eq!(lex.next(), Some(Ok(Token::Semicolon)));
222        assert_eq!(lex.next(), Some(Ok(Token::RBrace)));
223    }
224}
ternlang_core/lexer.rs

ternlang_core/
lexer.rs
