sage_parser/
token.rs

1//! Token definitions for the Sage lexer.
2
3use logos::Logos;
4
5/// All tokens in the Sage language.
6#[derive(Logos, Debug, Clone, PartialEq, Eq, Hash)]
7#[logos(skip r"[ \t\r\n]+")]
8#[logos(skip r"//[^\n]*")]
9pub enum Token {
10    // =========================================================================
11    // Keywords
12    // =========================================================================
13    #[token("agent")]
14    KwAgent,
15
16    #[token("belief")]
17    KwBelief,
18
19    #[token("on")]
20    KwOn,
21
22    #[token("start")]
23    KwStart,
24
25    #[token("stop")]
26    KwStop,
27
28    #[token("message")]
29    KwMessage,
30
31    #[token("divine")]
32    KwDivine,
33
34    #[token("summon")]
35    KwSummon,
36
37    #[token("await")]
38    KwAwait,
39
40    #[token("send")]
41    KwSend,
42
43    #[token("yield")]
44    KwYield,
45
46    #[token("run")]
47    KwRun,
48
49    #[token("fn")]
50    KwFn,
51
52    #[token("let")]
53    KwLet,
54
55    #[token("return")]
56    KwReturn,
57
58    #[token("if")]
59    KwIf,
60
61    #[token("else")]
62    KwElse,
63
64    #[token("for")]
65    KwFor,
66
67    #[token("while")]
68    KwWhile,
69
70    #[token("loop")]
71    KwLoop,
72
73    #[token("break")]
74    KwBreak,
75
76    #[token("in")]
77    KwIn,
78
79    #[token("self")]
80    KwSelf,
81
82    #[token("true")]
83    KwTrue,
84
85    #[token("false")]
86    KwFalse,
87
88    #[token("mod")]
89    KwMod,
90
91    #[token("use")]
92    KwUse,
93
94    #[token("pub")]
95    KwPub,
96
97    #[token("as")]
98    KwAs,
99
100    #[token("super")]
101    KwSuper,
102
103    #[token("record")]
104    KwRecord,
105
106    #[token("enum")]
107    KwEnum,
108
109    #[token("match")]
110    KwMatch,
111
112    #[token("const")]
113    KwConst,
114
115    #[token("receives")]
116    KwReceives,
117
118    #[token("receive")]
119    KwReceive,
120
121    #[token("fail")]
122    KwFail,
123
124    #[token("fails")]
125    KwFails,
126
127    #[token("timeout")]
128    KwTimeout,
129
130    #[token("retry")]
131    KwRetry,
132
133    #[token("delay")]
134    KwDelay,
135
136    #[token("try")]
137    KwTry,
138
139    #[token("catch")]
140    KwCatch,
141
142    #[token("error")]
143    KwError,
144
145    #[token("tool")]
146    KwTool,
147
148    /// RFC-0012: Test declaration keyword.
149    #[token("test")]
150    KwTest,
151
152    /// RFC-0012: Mock keyword for LLM mocking.
153    #[token("mock")]
154    KwMock,
155
156    /// Trace keyword for emitting trace events.
157    #[token("trace")]
158    KwTrace,
159
160    // =========================================================================
161    // Type keywords
162    // =========================================================================
163    #[token("Int")]
164    TyInt,
165
166    #[token("Float")]
167    TyFloat,
168
169    #[token("Bool")]
170    TyBool,
171
172    #[token("String")]
173    TyString,
174
175    #[token("Unit")]
176    TyUnit,
177
178    #[token("List")]
179    TyList,
180
181    #[token("Option")]
182    TyOption,
183
184    #[token("Oracle")]
185    TyOracle,
186
187    #[token("Agent")]
188    TyAgent,
189
190    #[token("Error")]
191    TyError,
192
193    #[token("ErrorKind")]
194    TyErrorKind,
195
196    /// Function type keyword: `Fn`
197    #[token("Fn")]
198    TyFn,
199
200    /// Map type keyword: `Map`
201    #[token("Map")]
202    TyMap,
203
204    /// Result type keyword: `Result`
205    #[token("Result")]
206    TyResult,
207
208    // =========================================================================
209    // Literals
210    // =========================================================================
211    /// Integer literal (e.g., `42`, `-7`).
212    #[regex(r"-?[0-9]+", priority = 2)]
213    IntLit,
214
215    /// Float literal (e.g., `3.14`, `-0.5`).
216    #[regex(r"-?[0-9]+\.[0-9]+")]
217    FloatLit,
218
219    /// String literal (e.g., `"hello"` or `'hello'`).
220    /// Supports escape sequences: \n, \t, \r, \\, \", \'
221    /// Both double and single quotes are allowed. Use single quotes inside
222    /// interpolations when needing string literals: `"Result: {len('hello')}"`
223    #[regex(r#""([^"\\]|\\.)*""#)]
224    #[regex(r#"'([^'\\]|\\.)*'"#)]
225    StringLit,
226
227    // =========================================================================
228    // Identifiers
229    // =========================================================================
230    /// Identifier (e.g., `foo`, `myAgent`, `_private`).
231    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*")]
232    Ident,
233
234    // =========================================================================
235    // Punctuation
236    // =========================================================================
237    #[token("{")]
238    LBrace,
239
240    #[token("}")]
241    RBrace,
242
243    #[token("(")]
244    LParen,
245
246    #[token(")")]
247    RParen,
248
249    #[token("[")]
250    LBracket,
251
252    #[token("]")]
253    RBracket,
254
255    #[token(",")]
256    Comma,
257
258    #[token("::")]
259    ColonColon,
260
261    #[token(":")]
262    Colon,
263
264    #[token(".")]
265    Dot,
266
267    #[token("->")]
268    Arrow,
269
270    #[token("=>")]
271    FatArrow,
272
273    /// Annotation marker for test attributes.
274    #[token("@")]
275    At,
276
277    // =========================================================================
278    // Operators
279    // =========================================================================
280    #[token("=")]
281    Eq,
282
283    #[token("==")]
284    EqEq,
285
286    #[token("!=")]
287    Ne,
288
289    #[token("<")]
290    Lt,
291
292    #[token(">")]
293    Gt,
294
295    #[token("<=")]
296    Le,
297
298    #[token(">=")]
299    Ge,
300
301    #[token("+")]
302    Plus,
303
304    #[token("-")]
305    Minus,
306
307    #[token("*")]
308    Star,
309
310    #[token("/")]
311    Slash,
312
313    #[token("!")]
314    Bang,
315
316    #[token("&&")]
317    And,
318
319    #[token("||")]
320    Or,
321
322    /// Single pipe for closure parameters: `|`
323    #[token("|")]
324    Pipe,
325
326    /// String concatenation operator.
327    #[token("++")]
328    PlusPlus,
329
330    /// Modulo/remainder operator.
331    #[token("%")]
332    Percent,
333
334    /// Statement terminator.
335    #[token(";")]
336    Semicolon,
337}
338
339impl Token {
340    /// Returns true if this token is a keyword.
341    #[must_use]
342    pub fn is_keyword(&self) -> bool {
343        matches!(
344            self,
345            Token::KwAgent
346                | Token::KwBelief
347                | Token::KwOn
348                | Token::KwStart
349                | Token::KwStop
350                | Token::KwMessage
351                | Token::KwDivine
352                | Token::KwSummon
353                | Token::KwAwait
354                | Token::KwSend
355                | Token::KwYield
356                | Token::KwRun
357                | Token::KwFn
358                | Token::KwLet
359                | Token::KwReturn
360                | Token::KwIf
361                | Token::KwElse
362                | Token::KwFor
363                | Token::KwWhile
364                | Token::KwLoop
365                | Token::KwBreak
366                | Token::KwIn
367                | Token::KwSelf
368                | Token::KwTrue
369                | Token::KwFalse
370                | Token::KwMod
371                | Token::KwUse
372                | Token::KwPub
373                | Token::KwAs
374                | Token::KwSuper
375                | Token::KwRecord
376                | Token::KwEnum
377                | Token::KwMatch
378                | Token::KwConst
379                | Token::KwReceives
380                | Token::KwReceive
381                | Token::KwFail
382                | Token::KwFails
383                | Token::KwTimeout
384                | Token::KwRetry
385                | Token::KwDelay
386                | Token::KwTry
387                | Token::KwCatch
388                | Token::KwError
389                | Token::KwTool
390                | Token::KwTrace
391        )
392    }
393
394    /// Returns true if this token is a type keyword.
395    #[must_use]
396    pub fn is_type_keyword(&self) -> bool {
397        matches!(
398            self,
399            Token::TyInt
400                | Token::TyFloat
401                | Token::TyBool
402                | Token::TyString
403                | Token::TyUnit
404                | Token::TyList
405                | Token::TyOption
406                | Token::TyOracle
407                | Token::TyAgent
408                | Token::TyError
409                | Token::TyErrorKind
410                | Token::TyFn
411                | Token::TyMap
412                | Token::TyResult
413        )
414    }
415
416    /// Returns true if this token is a literal.
417    #[must_use]
418    pub fn is_literal(&self) -> bool {
419        matches!(
420            self,
421            Token::IntLit | Token::FloatLit | Token::StringLit | Token::KwTrue | Token::KwFalse
422        )
423    }
424
425    /// Returns true if this token is an operator.
426    #[must_use]
427    pub fn is_operator(&self) -> bool {
428        matches!(
429            self,
430            Token::Eq
431                | Token::EqEq
432                | Token::Ne
433                | Token::Lt
434                | Token::Gt
435                | Token::Le
436                | Token::Ge
437                | Token::Plus
438                | Token::Minus
439                | Token::Star
440                | Token::Slash
441                | Token::Percent
442                | Token::Bang
443                | Token::And
444                | Token::Or
445                | Token::PlusPlus
446        )
447    }
448}
449
450impl std::fmt::Display for Token {
451    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
452        match self {
453            // Keywords
454            Token::KwAgent => write!(f, "agent"),
455            Token::KwBelief => write!(f, "belief"),
456            Token::KwOn => write!(f, "on"),
457            Token::KwStart => write!(f, "start"),
458            Token::KwStop => write!(f, "stop"),
459            Token::KwMessage => write!(f, "message"),
460            Token::KwDivine => write!(f, "divine"),
461            Token::KwSummon => write!(f, "summon"),
462            Token::KwAwait => write!(f, "await"),
463            Token::KwSend => write!(f, "send"),
464            Token::KwYield => write!(f, "yield"),
465            Token::KwRun => write!(f, "run"),
466            Token::KwFn => write!(f, "fn"),
467            Token::KwLet => write!(f, "let"),
468            Token::KwReturn => write!(f, "return"),
469            Token::KwIf => write!(f, "if"),
470            Token::KwElse => write!(f, "else"),
471            Token::KwFor => write!(f, "for"),
472            Token::KwWhile => write!(f, "while"),
473            Token::KwLoop => write!(f, "loop"),
474            Token::KwBreak => write!(f, "break"),
475            Token::KwIn => write!(f, "in"),
476            Token::KwSelf => write!(f, "self"),
477            Token::KwTrue => write!(f, "true"),
478            Token::KwFalse => write!(f, "false"),
479            Token::KwMod => write!(f, "mod"),
480            Token::KwUse => write!(f, "use"),
481            Token::KwPub => write!(f, "pub"),
482            Token::KwAs => write!(f, "as"),
483            Token::KwSuper => write!(f, "super"),
484            Token::KwRecord => write!(f, "record"),
485            Token::KwEnum => write!(f, "enum"),
486            Token::KwMatch => write!(f, "match"),
487            Token::KwConst => write!(f, "const"),
488            Token::KwReceives => write!(f, "receives"),
489            Token::KwReceive => write!(f, "receive"),
490            Token::KwFail => write!(f, "fail"),
491            Token::KwFails => write!(f, "fails"),
492            Token::KwTimeout => write!(f, "timeout"),
493            Token::KwRetry => write!(f, "retry"),
494            Token::KwDelay => write!(f, "delay"),
495            Token::KwTry => write!(f, "try"),
496            Token::KwCatch => write!(f, "catch"),
497            Token::KwError => write!(f, "error"),
498            Token::KwTool => write!(f, "tool"),
499            Token::KwTest => write!(f, "test"),
500            Token::KwMock => write!(f, "mock"),
501            Token::KwTrace => write!(f, "trace"),
502
503            // Type keywords
504            Token::TyInt => write!(f, "Int"),
505            Token::TyFloat => write!(f, "Float"),
506            Token::TyBool => write!(f, "Bool"),
507            Token::TyString => write!(f, "String"),
508            Token::TyUnit => write!(f, "Unit"),
509            Token::TyList => write!(f, "List"),
510            Token::TyOption => write!(f, "Option"),
511            Token::TyOracle => write!(f, "Oracle"),
512            Token::TyAgent => write!(f, "Agent"),
513            Token::TyError => write!(f, "Error"),
514            Token::TyErrorKind => write!(f, "ErrorKind"),
515            Token::TyFn => write!(f, "Fn"),
516            Token::TyMap => write!(f, "Map"),
517            Token::TyResult => write!(f, "Result"),
518
519            // Literals
520            Token::IntLit => write!(f, "<int>"),
521            Token::FloatLit => write!(f, "<float>"),
522            Token::StringLit => write!(f, "<string>"),
523
524            // Identifier
525            Token::Ident => write!(f, "<ident>"),
526
527            // Punctuation
528            Token::LBrace => write!(f, "{{"),
529            Token::RBrace => write!(f, "}}"),
530            Token::LParen => write!(f, "("),
531            Token::RParen => write!(f, ")"),
532            Token::LBracket => write!(f, "["),
533            Token::RBracket => write!(f, "]"),
534            Token::Comma => write!(f, ","),
535            Token::ColonColon => write!(f, "::"),
536            Token::Colon => write!(f, ":"),
537            Token::Dot => write!(f, "."),
538            Token::Arrow => write!(f, "->"),
539            Token::FatArrow => write!(f, "=>"),
540            Token::At => write!(f, "@"),
541
542            // Operators
543            Token::Eq => write!(f, "="),
544            Token::EqEq => write!(f, "=="),
545            Token::Ne => write!(f, "!="),
546            Token::Lt => write!(f, "<"),
547            Token::Gt => write!(f, ">"),
548            Token::Le => write!(f, "<="),
549            Token::Ge => write!(f, ">="),
550            Token::Plus => write!(f, "+"),
551            Token::Minus => write!(f, "-"),
552            Token::Star => write!(f, "*"),
553            Token::Slash => write!(f, "/"),
554            Token::Bang => write!(f, "!"),
555            Token::And => write!(f, "&&"),
556            Token::Or => write!(f, "||"),
557            Token::Pipe => write!(f, "|"),
558            Token::PlusPlus => write!(f, "++"),
559            Token::Percent => write!(f, "%"),
560            Token::Semicolon => write!(f, ";"),
561        }
562    }
563}
564
565#[cfg(test)]
566mod tests {
567    use super::*;
568
569    #[test]
570    fn lex_keywords() {
571        let mut lexer = Token::lexer("agent belief on start stop message");
572        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
573        assert_eq!(lexer.next(), Some(Ok(Token::KwBelief)));
574        assert_eq!(lexer.next(), Some(Ok(Token::KwOn)));
575        assert_eq!(lexer.next(), Some(Ok(Token::KwStart)));
576        assert_eq!(lexer.next(), Some(Ok(Token::KwStop)));
577        assert_eq!(lexer.next(), Some(Ok(Token::KwMessage)));
578        assert_eq!(lexer.next(), None);
579    }
580
581    #[test]
582    fn lex_more_keywords() {
583        let mut lexer = Token::lexer(
584            "divine summon await send yield run fn let return if else for in self true false",
585        );
586        assert_eq!(lexer.next(), Some(Ok(Token::KwDivine)));
587        assert_eq!(lexer.next(), Some(Ok(Token::KwSummon)));
588        assert_eq!(lexer.next(), Some(Ok(Token::KwAwait)));
589        assert_eq!(lexer.next(), Some(Ok(Token::KwSend)));
590        assert_eq!(lexer.next(), Some(Ok(Token::KwYield)));
591        assert_eq!(lexer.next(), Some(Ok(Token::KwRun)));
592        assert_eq!(lexer.next(), Some(Ok(Token::KwFn)));
593        assert_eq!(lexer.next(), Some(Ok(Token::KwLet)));
594        assert_eq!(lexer.next(), Some(Ok(Token::KwReturn)));
595        assert_eq!(lexer.next(), Some(Ok(Token::KwIf)));
596        assert_eq!(lexer.next(), Some(Ok(Token::KwElse)));
597        assert_eq!(lexer.next(), Some(Ok(Token::KwFor)));
598        assert_eq!(lexer.next(), Some(Ok(Token::KwIn)));
599        assert_eq!(lexer.next(), Some(Ok(Token::KwSelf)));
600        assert_eq!(lexer.next(), Some(Ok(Token::KwTrue)));
601        assert_eq!(lexer.next(), Some(Ok(Token::KwFalse)));
602        assert_eq!(lexer.next(), None);
603    }
604
605    #[test]
606    fn lex_type_keywords() {
607        let mut lexer = Token::lexer("Int Float Bool String Unit List Option Oracle Agent");
608        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
609        assert_eq!(lexer.next(), Some(Ok(Token::TyFloat)));
610        assert_eq!(lexer.next(), Some(Ok(Token::TyBool)));
611        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
612        assert_eq!(lexer.next(), Some(Ok(Token::TyUnit)));
613        assert_eq!(lexer.next(), Some(Ok(Token::TyList)));
614        assert_eq!(lexer.next(), Some(Ok(Token::TyOption)));
615        assert_eq!(lexer.next(), Some(Ok(Token::TyOracle)));
616        assert_eq!(lexer.next(), Some(Ok(Token::TyAgent)));
617        assert_eq!(lexer.next(), None);
618    }
619
620    #[test]
621    fn lex_integer_literals() {
622        let mut lexer = Token::lexer("42 -7 0 123456");
623        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
624        assert_eq!(lexer.slice(), "42");
625        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
626        assert_eq!(lexer.slice(), "-7");
627        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
628        assert_eq!(lexer.slice(), "0");
629        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
630        assert_eq!(lexer.slice(), "123456");
631        assert_eq!(lexer.next(), None);
632    }
633
634    #[test]
635    fn lex_float_literals() {
636        let mut lexer = Token::lexer("3.14 -0.5 0.0 123.456");
637        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
638        assert_eq!(lexer.slice(), "3.14");
639        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
640        assert_eq!(lexer.slice(), "-0.5");
641        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
642        assert_eq!(lexer.slice(), "0.0");
643        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
644        assert_eq!(lexer.slice(), "123.456");
645        assert_eq!(lexer.next(), None);
646    }
647
648    #[test]
649    fn lex_string_literals() {
650        let mut lexer = Token::lexer(r#""hello" "world" "with spaces""#);
651        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
652        assert_eq!(lexer.slice(), r#""hello""#);
653        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
654        assert_eq!(lexer.slice(), r#""world""#);
655        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
656        assert_eq!(lexer.slice(), r#""with spaces""#);
657        assert_eq!(lexer.next(), None);
658    }
659
660    #[test]
661    fn lex_string_with_escapes() {
662        let mut lexer = Token::lexer(r#""hello\nworld" "tab\there" "quote\"here""#);
663        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
664        assert_eq!(lexer.slice(), r#""hello\nworld""#);
665        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
666        assert_eq!(lexer.slice(), r#""tab\there""#);
667        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
668        assert_eq!(lexer.slice(), r#""quote\"here""#);
669        assert_eq!(lexer.next(), None);
670    }
671
672    #[test]
673    fn lex_identifiers() {
674        let mut lexer = Token::lexer("foo bar _private myAgent agent2");
675        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
676        assert_eq!(lexer.slice(), "foo");
677        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
678        assert_eq!(lexer.slice(), "bar");
679        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
680        assert_eq!(lexer.slice(), "_private");
681        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
682        assert_eq!(lexer.slice(), "myAgent");
683        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
684        assert_eq!(lexer.slice(), "agent2");
685        assert_eq!(lexer.next(), None);
686    }
687
688    #[test]
689    fn keyword_vs_identifier() {
690        // "agent" is a keyword, "agent_name" is an identifier
691        let mut lexer = Token::lexer("agent agent_name agents");
692        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
693        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
694        assert_eq!(lexer.slice(), "agent_name");
695        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
696        assert_eq!(lexer.slice(), "agents");
697        assert_eq!(lexer.next(), None);
698    }
699
700    #[test]
701    fn lex_punctuation() {
702        let mut lexer = Token::lexer("{ } ( ) [ ] , : . ->");
703        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
704        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
705        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
706        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
707        assert_eq!(lexer.next(), Some(Ok(Token::LBracket)));
708        assert_eq!(lexer.next(), Some(Ok(Token::RBracket)));
709        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
710        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
711        assert_eq!(lexer.next(), Some(Ok(Token::Dot)));
712        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
713        assert_eq!(lexer.next(), None);
714    }
715
716    #[test]
717    fn lex_operators() {
718        let mut lexer = Token::lexer("= == != < > <= >= + - * / % ! && || ++");
719        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
720        assert_eq!(lexer.next(), Some(Ok(Token::EqEq)));
721        assert_eq!(lexer.next(), Some(Ok(Token::Ne)));
722        assert_eq!(lexer.next(), Some(Ok(Token::Lt)));
723        assert_eq!(lexer.next(), Some(Ok(Token::Gt)));
724        assert_eq!(lexer.next(), Some(Ok(Token::Le)));
725        assert_eq!(lexer.next(), Some(Ok(Token::Ge)));
726        assert_eq!(lexer.next(), Some(Ok(Token::Plus)));
727        assert_eq!(lexer.next(), Some(Ok(Token::Minus)));
728        assert_eq!(lexer.next(), Some(Ok(Token::Star)));
729        assert_eq!(lexer.next(), Some(Ok(Token::Slash)));
730        assert_eq!(lexer.next(), Some(Ok(Token::Percent)));
731        assert_eq!(lexer.next(), Some(Ok(Token::Bang)));
732        assert_eq!(lexer.next(), Some(Ok(Token::And)));
733        assert_eq!(lexer.next(), Some(Ok(Token::Or)));
734        assert_eq!(lexer.next(), Some(Ok(Token::PlusPlus)));
735        assert_eq!(lexer.next(), None);
736    }
737
738    #[test]
739    fn skip_whitespace() {
740        let mut lexer = Token::lexer("  agent   belief\n\ttrue  ");
741        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
742        assert_eq!(lexer.next(), Some(Ok(Token::KwBelief)));
743        assert_eq!(lexer.next(), Some(Ok(Token::KwTrue)));
744        assert_eq!(lexer.next(), None);
745    }
746
747    #[test]
748    fn skip_comments() {
749        let mut lexer = Token::lexer("agent // this is a comment\nbelief");
750        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
751        assert_eq!(lexer.next(), Some(Ok(Token::KwBelief)));
752        assert_eq!(lexer.next(), None);
753    }
754
755    #[test]
756    fn comment_at_end() {
757        let mut lexer = Token::lexer("agent // comment at end");
758        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
759        assert_eq!(lexer.next(), None);
760    }
761
762    #[test]
763    fn lex_agent_declaration() {
764        let source = r#"
765            agent Researcher {
766                belief topic: String
767
768                on start {
769                    let result: Oracle<String> = divine("test")
770                    yield(result)
771                }
772            }
773        "#;
774        let tokens: Vec<_> = Token::lexer(source)
775            .map(|r| r.expect("valid token"))
776            .collect();
777
778        assert_eq!(tokens[0], Token::KwAgent);
779        assert_eq!(tokens[1], Token::Ident); // Researcher
780        assert_eq!(tokens[2], Token::LBrace);
781        assert_eq!(tokens[3], Token::KwBelief);
782        assert_eq!(tokens[4], Token::Ident); // topic
783        assert_eq!(tokens[5], Token::Colon);
784        assert_eq!(tokens[6], Token::TyString);
785        assert_eq!(tokens[7], Token::KwOn);
786        assert_eq!(tokens[8], Token::KwStart);
787        assert_eq!(tokens[9], Token::LBrace);
788        assert_eq!(tokens[10], Token::KwLet);
789    }
790
791    #[test]
792    fn is_keyword_helper() {
793        assert!(Token::KwAgent.is_keyword());
794        assert!(Token::KwLet.is_keyword());
795        assert!(!Token::TyInt.is_keyword());
796        assert!(!Token::Ident.is_keyword());
797    }
798
799    #[test]
800    fn is_type_keyword_helper() {
801        assert!(Token::TyInt.is_type_keyword());
802        assert!(Token::TyAgent.is_type_keyword());
803        assert!(!Token::KwAgent.is_type_keyword());
804        assert!(!Token::Ident.is_type_keyword());
805    }
806
807    #[test]
808    fn is_literal_helper() {
809        assert!(Token::IntLit.is_literal());
810        assert!(Token::FloatLit.is_literal());
811        assert!(Token::StringLit.is_literal());
812        assert!(Token::KwTrue.is_literal());
813        assert!(Token::KwFalse.is_literal());
814        assert!(!Token::Ident.is_literal());
815    }
816
817    #[test]
818    fn is_operator_helper() {
819        assert!(Token::Plus.is_operator());
820        assert!(Token::EqEq.is_operator());
821        assert!(Token::PlusPlus.is_operator());
822        assert!(!Token::LBrace.is_operator());
823        assert!(!Token::Ident.is_operator());
824    }
825
826    #[test]
827    fn lex_module_keywords() {
828        let mut lexer = Token::lexer("mod use pub as super");
829        assert_eq!(lexer.next(), Some(Ok(Token::KwMod)));
830        assert_eq!(lexer.next(), Some(Ok(Token::KwUse)));
831        assert_eq!(lexer.next(), Some(Ok(Token::KwPub)));
832        assert_eq!(lexer.next(), Some(Ok(Token::KwAs)));
833        assert_eq!(lexer.next(), Some(Ok(Token::KwSuper)));
834        assert_eq!(lexer.next(), None);
835    }
836
837    #[test]
838    fn lex_path_separator() {
839        let mut lexer = Token::lexer("agents::Researcher");
840        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
841        assert_eq!(lexer.slice(), "agents");
842        assert_eq!(lexer.next(), Some(Ok(Token::ColonColon)));
843        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
844        assert_eq!(lexer.slice(), "Researcher");
845        assert_eq!(lexer.next(), None);
846    }
847
848    #[test]
849    fn lex_use_statement() {
850        let mut lexer = Token::lexer("use agents::{Researcher, Coordinator as Coord}");
851        assert_eq!(lexer.next(), Some(Ok(Token::KwUse)));
852        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // agents
853        assert_eq!(lexer.next(), Some(Ok(Token::ColonColon)));
854        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
855        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Researcher
856        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
857        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Coordinator
858        assert_eq!(lexer.next(), Some(Ok(Token::KwAs)));
859        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Coord
860        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
861        assert_eq!(lexer.next(), None);
862    }
863
864    #[test]
865    fn lex_pub_agent() {
866        let mut lexer = Token::lexer("pub agent Researcher");
867        assert_eq!(lexer.next(), Some(Ok(Token::KwPub)));
868        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
869        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
870        assert_eq!(lexer.next(), None);
871    }
872
873    #[test]
874    fn token_display() {
875        assert_eq!(format!("{}", Token::KwAgent), "agent");
876        assert_eq!(format!("{}", Token::TyInt), "Int");
877        assert_eq!(format!("{}", Token::IntLit), "<int>");
878        assert_eq!(format!("{}", Token::Ident), "<ident>");
879        assert_eq!(format!("{}", Token::LBrace), "{");
880        assert_eq!(format!("{}", Token::PlusPlus), "++");
881    }
882
883    #[test]
884    fn lex_type_keywords_record_enum_match_const() {
885        let mut lexer = Token::lexer("record enum match const");
886        assert_eq!(lexer.next(), Some(Ok(Token::KwRecord)));
887        assert_eq!(lexer.next(), Some(Ok(Token::KwEnum)));
888        assert_eq!(lexer.next(), Some(Ok(Token::KwMatch)));
889        assert_eq!(lexer.next(), Some(Ok(Token::KwConst)));
890        assert_eq!(lexer.next(), None);
891    }
892
893    #[test]
894    fn lex_fat_arrow() {
895        let mut lexer = Token::lexer("=> -> =");
896        assert_eq!(lexer.next(), Some(Ok(Token::FatArrow)));
897        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
898        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
899        assert_eq!(lexer.next(), None);
900    }
901
902    #[test]
903    fn lex_match_expression() {
904        let mut lexer = Token::lexer("match status { Active => 1, Inactive => 0 }");
905        assert_eq!(lexer.next(), Some(Ok(Token::KwMatch)));
906        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // status
907        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
908        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Active
909        assert_eq!(lexer.next(), Some(Ok(Token::FatArrow)));
910        assert_eq!(lexer.next(), Some(Ok(Token::IntLit))); // 1
911        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
912        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Inactive
913        assert_eq!(lexer.next(), Some(Ok(Token::FatArrow)));
914        assert_eq!(lexer.next(), Some(Ok(Token::IntLit))); // 0
915        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
916        assert_eq!(lexer.next(), None);
917    }
918
919    #[test]
920    fn lex_record_declaration() {
921        let mut lexer = Token::lexer("record Point { x: Int, y: Int }");
922        assert_eq!(lexer.next(), Some(Ok(Token::KwRecord)));
923        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Point
924        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
925        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
926        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
927        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
928        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
929        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // y
930        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
931        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
932        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
933        assert_eq!(lexer.next(), None);
934    }
935
936    #[test]
937    fn lex_enum_declaration() {
938        let mut lexer = Token::lexer("enum Status { Active, Pending, Done }");
939        assert_eq!(lexer.next(), Some(Ok(Token::KwEnum)));
940        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Status
941        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
942        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Active
943        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
944        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Pending
945        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
946        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Done
947        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
948        assert_eq!(lexer.next(), None);
949    }
950
951    #[test]
952    fn lex_const_declaration() {
953        let mut lexer = Token::lexer("const MAX_RETRIES: Int = 3");
954        assert_eq!(lexer.next(), Some(Ok(Token::KwConst)));
955        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // MAX_RETRIES
956        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
957        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
958        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
959        assert_eq!(lexer.next(), Some(Ok(Token::IntLit))); // 3
960        assert_eq!(lexer.next(), None);
961    }
962
963    #[test]
964    fn new_keywords_are_keywords() {
965        assert!(Token::KwRecord.is_keyword());
966        assert!(Token::KwEnum.is_keyword());
967        assert!(Token::KwMatch.is_keyword());
968        assert!(Token::KwConst.is_keyword());
969    }
970
971    #[test]
972    fn lex_loop_break() {
973        let mut lexer = Token::lexer("loop { break }");
974        assert_eq!(lexer.next(), Some(Ok(Token::KwLoop)));
975        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
976        assert_eq!(lexer.next(), Some(Ok(Token::KwBreak)));
977        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
978        assert_eq!(lexer.next(), None);
979    }
980
981    #[test]
982    fn lex_receives_receive() {
983        let mut lexer = Token::lexer("agent Worker receives WorkerMsg { receive }");
984        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
985        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Worker
986        assert_eq!(lexer.next(), Some(Ok(Token::KwReceives)));
987        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // WorkerMsg
988        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
989        assert_eq!(lexer.next(), Some(Ok(Token::KwReceive)));
990        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
991        assert_eq!(lexer.next(), None);
992    }
993
994    #[test]
995    fn rfc6_keywords_are_keywords() {
996        assert!(Token::KwLoop.is_keyword());
997        assert!(Token::KwBreak.is_keyword());
998        assert!(Token::KwReceives.is_keyword());
999        assert!(Token::KwReceive.is_keyword());
1000    }
1001
1002    #[test]
1003    fn lex_error_handling_keywords() {
1004        let mut lexer = Token::lexer("fails try catch error");
1005        assert_eq!(lexer.next(), Some(Ok(Token::KwFails)));
1006        assert_eq!(lexer.next(), Some(Ok(Token::KwTry)));
1007        assert_eq!(lexer.next(), Some(Ok(Token::KwCatch)));
1008        assert_eq!(lexer.next(), Some(Ok(Token::KwError)));
1009        assert_eq!(lexer.next(), None);
1010    }
1011
1012    #[test]
1013    fn lex_try_catch_expression() {
1014        let mut lexer = Token::lexer("let x = try divine(prompt) catch { fallback }");
1015        assert_eq!(lexer.next(), Some(Ok(Token::KwLet)));
1016        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1017        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
1018        assert_eq!(lexer.next(), Some(Ok(Token::KwTry)));
1019        assert_eq!(lexer.next(), Some(Ok(Token::KwDivine)));
1020        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1021        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // prompt
1022        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1023        assert_eq!(lexer.next(), Some(Ok(Token::KwCatch)));
1024        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1025        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // fallback
1026        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1027        assert_eq!(lexer.next(), None);
1028    }
1029
1030    #[test]
1031    fn lex_fails_function() {
1032        let mut lexer = Token::lexer("fn fetch(url: String) -> String fails { }");
1033        assert_eq!(lexer.next(), Some(Ok(Token::KwFn)));
1034        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // fetch
1035        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1036        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // url
1037        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1038        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1039        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1040        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1041        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1042        assert_eq!(lexer.next(), Some(Ok(Token::KwFails)));
1043        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1044        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1045        assert_eq!(lexer.next(), None);
1046    }
1047
1048    #[test]
1049    fn lex_on_error_handler() {
1050        let mut lexer = Token::lexer("on error(e) { yield(fallback) }");
1051        assert_eq!(lexer.next(), Some(Ok(Token::KwOn)));
1052        assert_eq!(lexer.next(), Some(Ok(Token::KwError)));
1053        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1054        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // e
1055        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1056        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1057        assert_eq!(lexer.next(), Some(Ok(Token::KwYield)));
1058        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1059        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // fallback
1060        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1061        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1062        assert_eq!(lexer.next(), None);
1063    }
1064
1065    #[test]
1066    fn rfc7_keywords_are_keywords() {
1067        assert!(Token::KwFail.is_keyword());
1068        assert!(Token::KwFails.is_keyword());
1069        assert!(Token::KwTry.is_keyword());
1070        assert!(Token::KwCatch.is_keyword());
1071        assert!(Token::KwError.is_keyword());
1072    }
1073
1074    #[test]
1075    fn lex_fail_expression() {
1076        let mut lexer = Token::lexer("fail \"error message\"");
1077        assert_eq!(lexer.next(), Some(Ok(Token::KwFail)));
1078        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
1079        assert_eq!(lexer.next(), None);
1080    }
1081
1082    // =========================================================================
1083    // RFC-0009: Closures
1084    // =========================================================================
1085
1086    #[test]
1087    fn lex_closure_syntax() {
1088        // |x: Int| x + 1
1089        let mut lexer = Token::lexer("|x: Int| x + 1");
1090        assert_eq!(lexer.next(), Some(Ok(Token::Pipe)));
1091        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1092        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1093        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1094        assert_eq!(lexer.next(), Some(Ok(Token::Pipe)));
1095        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1096        assert_eq!(lexer.next(), Some(Ok(Token::Plus)));
1097        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
1098        assert_eq!(lexer.next(), None);
1099    }
1100
1101    #[test]
1102    fn lex_empty_closure() {
1103        // || 42
1104        let mut lexer = Token::lexer("|| 42");
1105        assert_eq!(lexer.next(), Some(Ok(Token::Or))); // || lexes as Or
1106        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
1107        assert_eq!(lexer.next(), None);
1108    }
1109
1110    #[test]
1111    fn lex_fn_type() {
1112        // Fn(Int, String) -> Bool
1113        let mut lexer = Token::lexer("Fn(Int, String) -> Bool");
1114        assert_eq!(lexer.next(), Some(Ok(Token::TyFn)));
1115        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1116        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1117        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
1118        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1119        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1120        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1121        assert_eq!(lexer.next(), Some(Ok(Token::TyBool)));
1122        assert_eq!(lexer.next(), None);
1123    }
1124
1125    #[test]
1126    fn fn_is_type_keyword() {
1127        assert!(Token::TyFn.is_type_keyword());
1128    }
1129
1130    #[test]
1131    fn pipe_display() {
1132        assert_eq!(format!("{}", Token::Pipe), "|");
1133        assert_eq!(format!("{}", Token::TyFn), "Fn");
1134    }
1135
1136    // =========================================================================
1137    // RFC-0011: Tool Support
1138    // =========================================================================
1139
1140    #[test]
1141    fn lex_tool_keyword() {
1142        let mut lexer = Token::lexer("tool Http { fn get(url: String) -> String }");
1143        assert_eq!(lexer.next(), Some(Ok(Token::KwTool)));
1144        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Http
1145        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1146        assert_eq!(lexer.next(), Some(Ok(Token::KwFn)));
1147        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // get
1148        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1149        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // url
1150        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1151        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1152        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1153        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1154        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1155        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1156        assert_eq!(lexer.next(), None);
1157    }
1158
1159    #[test]
1160    fn tool_is_keyword() {
1161        assert!(Token::KwTool.is_keyword());
1162    }
1163
1164    #[test]
1165    fn lex_agent_use_tool() {
1166        let mut lexer = Token::lexer("agent Fetcher { use Http }");
1167        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
1168        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Fetcher
1169        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1170        assert_eq!(lexer.next(), Some(Ok(Token::KwUse)));
1171        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Http
1172        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1173        assert_eq!(lexer.next(), None);
1174    }
1175}
sage_parser/token.rs

sage_parser/
token.rs
