sage_parser/
token.rs

1//! Token definitions for the Sage lexer.
2
3use logos::Logos;
4
5/// All tokens in the Sage language.
6#[derive(Logos, Debug, Clone, PartialEq, Eq, Hash)]
7#[logos(skip r"[ \t\r\n]+")]
8#[logos(skip r"//[^\n]*")]
9pub enum Token {
10    // =========================================================================
11    // Keywords
12    // =========================================================================
13    #[token("agent")]
14    KwAgent,
15
16    #[token("belief")]
17    KwBelief,
18
19    #[token("on")]
20    KwOn,
21
22    #[token("start")]
23    KwStart,
24
25    #[token("stop")]
26    KwStop,
27
28    /// v2 lifecycle: runs before start, after persistent state loaded.
29    #[token("waking")]
30    KwWaking,
31
32    /// v2 lifecycle: runs when supervisor signals graceful pause.
33    #[token("pause")]
34    KwPause,
35
36    /// v2 lifecycle: runs when agent is unpaused.
37    #[token("resume")]
38    KwResume,
39
40    /// v2 lifecycle: alias for stop (v2 terminology).
41    #[token("resting")]
42    KwResting,
43
44    #[token("message")]
45    KwMessage,
46
47    #[token("divine")]
48    KwDivine,
49
50    #[token("infer")]
51    KwInfer,
52
53    #[token("summon")]
54    KwSummon,
55
56    #[token("await")]
57    KwAwait,
58
59    #[token("send")]
60    KwSend,
61
62    #[token("yield")]
63    KwYield,
64
65    #[token("run")]
66    KwRun,
67
68    #[token("fn")]
69    KwFn,
70
71    #[token("let")]
72    KwLet,
73
74    #[token("return")]
75    KwReturn,
76
77    #[token("if")]
78    KwIf,
79
80    #[token("else")]
81    KwElse,
82
83    #[token("for")]
84    KwFor,
85
86    #[token("while")]
87    KwWhile,
88
89    #[token("loop")]
90    KwLoop,
91
92    #[token("break")]
93    KwBreak,
94
95    #[token("in")]
96    KwIn,
97
98    #[token("self")]
99    KwSelf,
100
101    #[token("true")]
102    KwTrue,
103
104    #[token("false")]
105    KwFalse,
106
107    #[token("mod")]
108    KwMod,
109
110    #[token("use")]
111    KwUse,
112
113    #[token("pub")]
114    KwPub,
115
116    #[token("as")]
117    KwAs,
118
119    #[token("super")]
120    KwSuper,
121
122    #[token("record")]
123    KwRecord,
124
125    #[token("enum")]
126    KwEnum,
127
128    #[token("match")]
129    KwMatch,
130
131    #[token("const")]
132    KwConst,
133
134    #[token("receives")]
135    KwReceives,
136
137    #[token("receive")]
138    KwReceive,
139
140    #[token("fail")]
141    KwFail,
142
143    #[token("fails")]
144    KwFails,
145
146    #[token("timeout")]
147    KwTimeout,
148
149    #[token("retry")]
150    KwRetry,
151
152    #[token("delay")]
153    KwDelay,
154
155    #[token("try")]
156    KwTry,
157
158    #[token("catch")]
159    KwCatch,
160
161    #[token("error")]
162    KwError,
163
164    #[token("tool")]
165    KwTool,
166
167    /// RFC-0012: Test declaration keyword.
168    #[token("test")]
169    KwTest,
170
171    /// RFC-0012: Mock keyword for LLM mocking.
172    #[token("mock")]
173    KwMock,
174
175    /// Trace keyword for emitting trace events.
176    #[token("trace")]
177    KwTrace,
178
179    /// Span keyword for timed observability blocks.
180    #[token("span")]
181    KwSpan,
182
183    /// Checkpoint keyword for explicit persistence checkpoint.
184    #[token("checkpoint")]
185    KwCheckpoint,
186
187    // =========================================================================
188    // Supervision tree keywords (v2)
189    // =========================================================================
190    /// Supervisor declaration keyword.
191    #[token("supervisor")]
192    KwSupervisor,
193
194    /// Children block in supervisor.
195    #[token("children")]
196    KwChildren,
197
198    /// Supervision strategy keyword.
199    #[token("strategy")]
200    KwStrategy,
201
202    /// Restart policy keyword.
203    #[token("restart")]
204    KwRestart,
205
206    // =========================================================================
207    // Session types keywords (Phase 3)
208    // =========================================================================
209    /// Protocol declaration keyword.
210    #[token("protocol")]
211    KwProtocol,
212
213    /// Follows clause in agent declaration.
214    #[token("follows")]
215    KwFollows,
216
217    /// Reply expression in message handlers.
218    #[token("reply")]
219    KwReply,
220
221    // =========================================================================
222    // Algebraic effects keywords (Phase 3)
223    // =========================================================================
224    /// Effect handler declaration keyword.
225    #[token("handler")]
226    KwHandler,
227
228    /// Handles clause in effect handler declaration.
229    #[token("handles")]
230    KwHandles,
231
232    // =========================================================================
233    // Extern (Rust FFI) keywords
234    // =========================================================================
235    /// Extern keyword for Rust FFI function declarations.
236    #[token("extern")]
237    KwExtern,
238
239    // =========================================================================
240    // Type keywords
241    // =========================================================================
242    #[token("Int")]
243    TyInt,
244
245    #[token("Float")]
246    TyFloat,
247
248    #[token("Bool")]
249    TyBool,
250
251    #[token("String")]
252    TyString,
253
254    #[token("Unit")]
255    TyUnit,
256
257    #[token("List")]
258    TyList,
259
260    #[token("Option")]
261    TyOption,
262
263    #[token("Oracle")]
264    TyOracle,
265
266    #[token("Agent")]
267    TyAgent,
268
269    #[token("Error")]
270    TyError,
271
272    #[token("ErrorKind")]
273    TyErrorKind,
274
275    /// Function type keyword: `Fn`
276    #[token("Fn")]
277    TyFn,
278
279    /// Map type keyword: `Map`
280    #[token("Map")]
281    TyMap,
282
283    /// Result type keyword: `Result`
284    #[token("Result")]
285    TyResult,
286
287    // =========================================================================
288    // Literals
289    // =========================================================================
290    /// Integer literal (e.g., `42`, `-7`).
291    #[regex(r"-?[0-9]+", priority = 2)]
292    IntLit,
293
294    /// Float literal (e.g., `3.14`, `-0.5`).
295    #[regex(r"-?[0-9]+\.[0-9]+")]
296    FloatLit,
297
298    /// String literal (e.g., `"hello"` or `'hello'`).
299    /// Supports escape sequences: \n, \t, \r, \\, \", \', \xNN (hex)
300    /// Both double and single quotes are allowed. Use single quotes inside
301    /// interpolations when needing string literals: `"Result: {len('hello')}"`
302    #[regex(r#""([^"\\]|\\.)*""#)]
303    #[regex(r#"'([^'\\]|\\.)*'"#)]
304    StringLit,
305
306    // =========================================================================
307    // Identifiers
308    // =========================================================================
309    /// Identifier (e.g., `foo`, `myAgent`, `_private`).
310    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*")]
311    Ident,
312
313    // =========================================================================
314    // Punctuation
315    // =========================================================================
316    #[token("{")]
317    LBrace,
318
319    #[token("}")]
320    RBrace,
321
322    #[token("(")]
323    LParen,
324
325    #[token(")")]
326    RParen,
327
328    #[token("[")]
329    LBracket,
330
331    #[token("]")]
332    RBracket,
333
334    #[token(",")]
335    Comma,
336
337    #[token("::")]
338    ColonColon,
339
340    #[token(":")]
341    Colon,
342
343    #[token(".")]
344    Dot,
345
346    #[token("->")]
347    Arrow,
348
349    #[token("=>")]
350    FatArrow,
351
352    /// Annotation marker for test attributes.
353    #[token("@")]
354    At,
355
356    // =========================================================================
357    // Operators
358    // =========================================================================
359    #[token("=")]
360    Eq,
361
362    #[token("==")]
363    EqEq,
364
365    #[token("!=")]
366    Ne,
367
368    #[token("<")]
369    Lt,
370
371    #[token(">")]
372    Gt,
373
374    #[token("<=")]
375    Le,
376
377    #[token(">=")]
378    Ge,
379
380    #[token("+")]
381    Plus,
382
383    #[token("-")]
384    Minus,
385
386    #[token("*")]
387    Star,
388
389    #[token("/")]
390    Slash,
391
392    #[token("!")]
393    Bang,
394
395    #[token("&&")]
396    And,
397
398    #[token("||")]
399    Or,
400
401    /// Single pipe for closure parameters: `|`
402    #[token("|")]
403    Pipe,
404
405    /// String concatenation operator.
406    #[token("++")]
407    PlusPlus,
408
409    /// Modulo/remainder operator.
410    #[token("%")]
411    Percent,
412
413    /// Statement terminator.
414    #[token(";")]
415    Semicolon,
416}
417
418impl Token {
419    /// Returns true if this token is a keyword.
420    #[must_use]
421    pub fn is_keyword(&self) -> bool {
422        matches!(
423            self,
424            Token::KwAgent
425                | Token::KwBelief
426                | Token::KwOn
427                | Token::KwStart
428                | Token::KwStop
429                | Token::KwWaking
430                | Token::KwPause
431                | Token::KwResume
432                | Token::KwResting
433                | Token::KwMessage
434                | Token::KwDivine
435                | Token::KwInfer
436                | Token::KwSummon
437                | Token::KwAwait
438                | Token::KwSend
439                | Token::KwYield
440                | Token::KwRun
441                | Token::KwFn
442                | Token::KwLet
443                | Token::KwReturn
444                | Token::KwIf
445                | Token::KwElse
446                | Token::KwFor
447                | Token::KwWhile
448                | Token::KwLoop
449                | Token::KwBreak
450                | Token::KwIn
451                | Token::KwSelf
452                | Token::KwTrue
453                | Token::KwFalse
454                | Token::KwMod
455                | Token::KwUse
456                | Token::KwPub
457                | Token::KwAs
458                | Token::KwSuper
459                | Token::KwRecord
460                | Token::KwEnum
461                | Token::KwMatch
462                | Token::KwConst
463                | Token::KwReceives
464                | Token::KwReceive
465                | Token::KwFail
466                | Token::KwFails
467                | Token::KwTimeout
468                | Token::KwRetry
469                | Token::KwDelay
470                | Token::KwTry
471                | Token::KwCatch
472                | Token::KwError
473                | Token::KwTool
474                | Token::KwTrace
475                | Token::KwSpan
476                | Token::KwCheckpoint
477                | Token::KwSupervisor
478                | Token::KwChildren
479                | Token::KwStrategy
480                | Token::KwRestart
481                | Token::KwProtocol
482                | Token::KwFollows
483                | Token::KwReply
484                | Token::KwHandler
485                | Token::KwHandles
486                | Token::KwExtern
487        )
488    }
489
490    /// Returns true if this token is a type keyword.
491    #[must_use]
492    pub fn is_type_keyword(&self) -> bool {
493        matches!(
494            self,
495            Token::TyInt
496                | Token::TyFloat
497                | Token::TyBool
498                | Token::TyString
499                | Token::TyUnit
500                | Token::TyList
501                | Token::TyOption
502                | Token::TyOracle
503                | Token::TyAgent
504                | Token::TyError
505                | Token::TyErrorKind
506                | Token::TyFn
507                | Token::TyMap
508                | Token::TyResult
509        )
510    }
511
512    /// Returns true if this token is a literal.
513    #[must_use]
514    pub fn is_literal(&self) -> bool {
515        matches!(
516            self,
517            Token::IntLit | Token::FloatLit | Token::StringLit | Token::KwTrue | Token::KwFalse
518        )
519    }
520
521    /// Returns true if this token is an operator.
522    #[must_use]
523    pub fn is_operator(&self) -> bool {
524        matches!(
525            self,
526            Token::Eq
527                | Token::EqEq
528                | Token::Ne
529                | Token::Lt
530                | Token::Gt
531                | Token::Le
532                | Token::Ge
533                | Token::Plus
534                | Token::Minus
535                | Token::Star
536                | Token::Slash
537                | Token::Percent
538                | Token::Bang
539                | Token::And
540                | Token::Or
541                | Token::PlusPlus
542        )
543    }
544}
545
546impl std::fmt::Display for Token {
547    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
548        match self {
549            // Keywords
550            Token::KwAgent => write!(f, "agent"),
551            Token::KwBelief => write!(f, "belief"),
552            Token::KwOn => write!(f, "on"),
553            Token::KwStart => write!(f, "start"),
554            Token::KwStop => write!(f, "stop"),
555            Token::KwWaking => write!(f, "waking"),
556            Token::KwPause => write!(f, "pause"),
557            Token::KwResume => write!(f, "resume"),
558            Token::KwResting => write!(f, "resting"),
559            Token::KwMessage => write!(f, "message"),
560            Token::KwDivine => write!(f, "divine"),
561            Token::KwInfer => write!(f, "infer"),
562            Token::KwSummon => write!(f, "summon"),
563            Token::KwAwait => write!(f, "await"),
564            Token::KwSend => write!(f, "send"),
565            Token::KwYield => write!(f, "yield"),
566            Token::KwRun => write!(f, "run"),
567            Token::KwFn => write!(f, "fn"),
568            Token::KwLet => write!(f, "let"),
569            Token::KwReturn => write!(f, "return"),
570            Token::KwIf => write!(f, "if"),
571            Token::KwElse => write!(f, "else"),
572            Token::KwFor => write!(f, "for"),
573            Token::KwWhile => write!(f, "while"),
574            Token::KwLoop => write!(f, "loop"),
575            Token::KwBreak => write!(f, "break"),
576            Token::KwIn => write!(f, "in"),
577            Token::KwSelf => write!(f, "self"),
578            Token::KwTrue => write!(f, "true"),
579            Token::KwFalse => write!(f, "false"),
580            Token::KwMod => write!(f, "mod"),
581            Token::KwUse => write!(f, "use"),
582            Token::KwPub => write!(f, "pub"),
583            Token::KwAs => write!(f, "as"),
584            Token::KwSuper => write!(f, "super"),
585            Token::KwRecord => write!(f, "record"),
586            Token::KwEnum => write!(f, "enum"),
587            Token::KwMatch => write!(f, "match"),
588            Token::KwConst => write!(f, "const"),
589            Token::KwReceives => write!(f, "receives"),
590            Token::KwReceive => write!(f, "receive"),
591            Token::KwFail => write!(f, "fail"),
592            Token::KwFails => write!(f, "fails"),
593            Token::KwTimeout => write!(f, "timeout"),
594            Token::KwRetry => write!(f, "retry"),
595            Token::KwDelay => write!(f, "delay"),
596            Token::KwTry => write!(f, "try"),
597            Token::KwCatch => write!(f, "catch"),
598            Token::KwError => write!(f, "error"),
599            Token::KwTool => write!(f, "tool"),
600            Token::KwTest => write!(f, "test"),
601            Token::KwMock => write!(f, "mock"),
602            Token::KwTrace => write!(f, "trace"),
603            Token::KwSpan => write!(f, "span"),
604            Token::KwCheckpoint => write!(f, "checkpoint"),
605            Token::KwSupervisor => write!(f, "supervisor"),
606            Token::KwChildren => write!(f, "children"),
607            Token::KwStrategy => write!(f, "strategy"),
608            Token::KwRestart => write!(f, "restart"),
609            Token::KwProtocol => write!(f, "protocol"),
610            Token::KwFollows => write!(f, "follows"),
611            Token::KwReply => write!(f, "reply"),
612            Token::KwHandler => write!(f, "handler"),
613            Token::KwHandles => write!(f, "handles"),
614            Token::KwExtern => write!(f, "extern"),
615
616            // Type keywords
617            Token::TyInt => write!(f, "Int"),
618            Token::TyFloat => write!(f, "Float"),
619            Token::TyBool => write!(f, "Bool"),
620            Token::TyString => write!(f, "String"),
621            Token::TyUnit => write!(f, "Unit"),
622            Token::TyList => write!(f, "List"),
623            Token::TyOption => write!(f, "Option"),
624            Token::TyOracle => write!(f, "Oracle"),
625            Token::TyAgent => write!(f, "Agent"),
626            Token::TyError => write!(f, "Error"),
627            Token::TyErrorKind => write!(f, "ErrorKind"),
628            Token::TyFn => write!(f, "Fn"),
629            Token::TyMap => write!(f, "Map"),
630            Token::TyResult => write!(f, "Result"),
631
632            // Literals
633            Token::IntLit => write!(f, "<int>"),
634            Token::FloatLit => write!(f, "<float>"),
635            Token::StringLit => write!(f, "<string>"),
636
637            // Identifier
638            Token::Ident => write!(f, "<ident>"),
639
640            // Punctuation
641            Token::LBrace => write!(f, "{{"),
642            Token::RBrace => write!(f, "}}"),
643            Token::LParen => write!(f, "("),
644            Token::RParen => write!(f, ")"),
645            Token::LBracket => write!(f, "["),
646            Token::RBracket => write!(f, "]"),
647            Token::Comma => write!(f, ","),
648            Token::ColonColon => write!(f, "::"),
649            Token::Colon => write!(f, ":"),
650            Token::Dot => write!(f, "."),
651            Token::Arrow => write!(f, "->"),
652            Token::FatArrow => write!(f, "=>"),
653            Token::At => write!(f, "@"),
654
655            // Operators
656            Token::Eq => write!(f, "="),
657            Token::EqEq => write!(f, "=="),
658            Token::Ne => write!(f, "!="),
659            Token::Lt => write!(f, "<"),
660            Token::Gt => write!(f, ">"),
661            Token::Le => write!(f, "<="),
662            Token::Ge => write!(f, ">="),
663            Token::Plus => write!(f, "+"),
664            Token::Minus => write!(f, "-"),
665            Token::Star => write!(f, "*"),
666            Token::Slash => write!(f, "/"),
667            Token::Bang => write!(f, "!"),
668            Token::And => write!(f, "&&"),
669            Token::Or => write!(f, "||"),
670            Token::Pipe => write!(f, "|"),
671            Token::PlusPlus => write!(f, "++"),
672            Token::Percent => write!(f, "%"),
673            Token::Semicolon => write!(f, ";"),
674        }
675    }
676}
677
678#[cfg(test)]
679mod tests {
680    use super::*;
681
682    #[test]
683    fn lex_keywords() {
684        let mut lexer = Token::lexer("agent belief on start stop message");
685        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
686        assert_eq!(lexer.next(), Some(Ok(Token::KwBelief)));
687        assert_eq!(lexer.next(), Some(Ok(Token::KwOn)));
688        assert_eq!(lexer.next(), Some(Ok(Token::KwStart)));
689        assert_eq!(lexer.next(), Some(Ok(Token::KwStop)));
690        assert_eq!(lexer.next(), Some(Ok(Token::KwMessage)));
691        assert_eq!(lexer.next(), None);
692    }
693
694    #[test]
695    fn lex_more_keywords() {
696        let mut lexer = Token::lexer(
697            "divine summon await send yield run fn let return if else for in self true false",
698        );
699        assert_eq!(lexer.next(), Some(Ok(Token::KwDivine)));
700        assert_eq!(lexer.next(), Some(Ok(Token::KwSummon)));
701        assert_eq!(lexer.next(), Some(Ok(Token::KwAwait)));
702        assert_eq!(lexer.next(), Some(Ok(Token::KwSend)));
703        assert_eq!(lexer.next(), Some(Ok(Token::KwYield)));
704        assert_eq!(lexer.next(), Some(Ok(Token::KwRun)));
705        assert_eq!(lexer.next(), Some(Ok(Token::KwFn)));
706        assert_eq!(lexer.next(), Some(Ok(Token::KwLet)));
707        assert_eq!(lexer.next(), Some(Ok(Token::KwReturn)));
708        assert_eq!(lexer.next(), Some(Ok(Token::KwIf)));
709        assert_eq!(lexer.next(), Some(Ok(Token::KwElse)));
710        assert_eq!(lexer.next(), Some(Ok(Token::KwFor)));
711        assert_eq!(lexer.next(), Some(Ok(Token::KwIn)));
712        assert_eq!(lexer.next(), Some(Ok(Token::KwSelf)));
713        assert_eq!(lexer.next(), Some(Ok(Token::KwTrue)));
714        assert_eq!(lexer.next(), Some(Ok(Token::KwFalse)));
715        assert_eq!(lexer.next(), None);
716    }
717
718    #[test]
719    fn lex_type_keywords() {
720        let mut lexer = Token::lexer("Int Float Bool String Unit List Option Oracle Agent");
721        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
722        assert_eq!(lexer.next(), Some(Ok(Token::TyFloat)));
723        assert_eq!(lexer.next(), Some(Ok(Token::TyBool)));
724        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
725        assert_eq!(lexer.next(), Some(Ok(Token::TyUnit)));
726        assert_eq!(lexer.next(), Some(Ok(Token::TyList)));
727        assert_eq!(lexer.next(), Some(Ok(Token::TyOption)));
728        assert_eq!(lexer.next(), Some(Ok(Token::TyOracle)));
729        assert_eq!(lexer.next(), Some(Ok(Token::TyAgent)));
730        assert_eq!(lexer.next(), None);
731    }
732
733    #[test]
734    fn lex_integer_literals() {
735        let mut lexer = Token::lexer("42 -7 0 123456");
736        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
737        assert_eq!(lexer.slice(), "42");
738        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
739        assert_eq!(lexer.slice(), "-7");
740        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
741        assert_eq!(lexer.slice(), "0");
742        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
743        assert_eq!(lexer.slice(), "123456");
744        assert_eq!(lexer.next(), None);
745    }
746
747    #[test]
748    fn lex_float_literals() {
749        let mut lexer = Token::lexer("3.14 -0.5 0.0 123.456");
750        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
751        assert_eq!(lexer.slice(), "3.14");
752        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
753        assert_eq!(lexer.slice(), "-0.5");
754        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
755        assert_eq!(lexer.slice(), "0.0");
756        assert_eq!(lexer.next(), Some(Ok(Token::FloatLit)));
757        assert_eq!(lexer.slice(), "123.456");
758        assert_eq!(lexer.next(), None);
759    }
760
761    #[test]
762    fn lex_string_literals() {
763        let mut lexer = Token::lexer(r#""hello" "world" "with spaces""#);
764        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
765        assert_eq!(lexer.slice(), r#""hello""#);
766        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
767        assert_eq!(lexer.slice(), r#""world""#);
768        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
769        assert_eq!(lexer.slice(), r#""with spaces""#);
770        assert_eq!(lexer.next(), None);
771    }
772
773    #[test]
774    fn lex_string_with_escapes() {
775        let mut lexer = Token::lexer(r#""hello\nworld" "tab\there" "quote\"here""#);
776        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
777        assert_eq!(lexer.slice(), r#""hello\nworld""#);
778        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
779        assert_eq!(lexer.slice(), r#""tab\there""#);
780        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
781        assert_eq!(lexer.slice(), r#""quote\"here""#);
782        assert_eq!(lexer.next(), None);
783    }
784
785    #[test]
786    fn lex_identifiers() {
787        let mut lexer = Token::lexer("foo bar _private myAgent agent2");
788        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
789        assert_eq!(lexer.slice(), "foo");
790        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
791        assert_eq!(lexer.slice(), "bar");
792        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
793        assert_eq!(lexer.slice(), "_private");
794        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
795        assert_eq!(lexer.slice(), "myAgent");
796        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
797        assert_eq!(lexer.slice(), "agent2");
798        assert_eq!(lexer.next(), None);
799    }
800
801    #[test]
802    fn keyword_vs_identifier() {
803        // "agent" is a keyword, "agent_name" is an identifier
804        let mut lexer = Token::lexer("agent agent_name agents");
805        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
806        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
807        assert_eq!(lexer.slice(), "agent_name");
808        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
809        assert_eq!(lexer.slice(), "agents");
810        assert_eq!(lexer.next(), None);
811    }
812
813    #[test]
814    fn lex_punctuation() {
815        let mut lexer = Token::lexer("{ } ( ) [ ] , : . ->");
816        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
817        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
818        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
819        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
820        assert_eq!(lexer.next(), Some(Ok(Token::LBracket)));
821        assert_eq!(lexer.next(), Some(Ok(Token::RBracket)));
822        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
823        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
824        assert_eq!(lexer.next(), Some(Ok(Token::Dot)));
825        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
826        assert_eq!(lexer.next(), None);
827    }
828
829    #[test]
830    fn lex_operators() {
831        let mut lexer = Token::lexer("= == != < > <= >= + - * / % ! && || ++");
832        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
833        assert_eq!(lexer.next(), Some(Ok(Token::EqEq)));
834        assert_eq!(lexer.next(), Some(Ok(Token::Ne)));
835        assert_eq!(lexer.next(), Some(Ok(Token::Lt)));
836        assert_eq!(lexer.next(), Some(Ok(Token::Gt)));
837        assert_eq!(lexer.next(), Some(Ok(Token::Le)));
838        assert_eq!(lexer.next(), Some(Ok(Token::Ge)));
839        assert_eq!(lexer.next(), Some(Ok(Token::Plus)));
840        assert_eq!(lexer.next(), Some(Ok(Token::Minus)));
841        assert_eq!(lexer.next(), Some(Ok(Token::Star)));
842        assert_eq!(lexer.next(), Some(Ok(Token::Slash)));
843        assert_eq!(lexer.next(), Some(Ok(Token::Percent)));
844        assert_eq!(lexer.next(), Some(Ok(Token::Bang)));
845        assert_eq!(lexer.next(), Some(Ok(Token::And)));
846        assert_eq!(lexer.next(), Some(Ok(Token::Or)));
847        assert_eq!(lexer.next(), Some(Ok(Token::PlusPlus)));
848        assert_eq!(lexer.next(), None);
849    }
850
851    #[test]
852    fn skip_whitespace() {
853        let mut lexer = Token::lexer("  agent   belief\n\ttrue  ");
854        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
855        assert_eq!(lexer.next(), Some(Ok(Token::KwBelief)));
856        assert_eq!(lexer.next(), Some(Ok(Token::KwTrue)));
857        assert_eq!(lexer.next(), None);
858    }
859
860    #[test]
861    fn skip_comments() {
862        let mut lexer = Token::lexer("agent // this is a comment\nbelief");
863        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
864        assert_eq!(lexer.next(), Some(Ok(Token::KwBelief)));
865        assert_eq!(lexer.next(), None);
866    }
867
868    #[test]
869    fn comment_at_end() {
870        let mut lexer = Token::lexer("agent // comment at end");
871        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
872        assert_eq!(lexer.next(), None);
873    }
874
875    #[test]
876    fn lex_agent_declaration() {
877        let source = r#"
878            agent Researcher {
879                belief topic: String
880
881                on start {
882                    let result: Oracle<String> = divine("test")
883                    yield(result)
884                }
885            }
886        "#;
887        let tokens: Vec<_> = Token::lexer(source)
888            .map(|r| r.expect("valid token"))
889            .collect();
890
891        assert_eq!(tokens[0], Token::KwAgent);
892        assert_eq!(tokens[1], Token::Ident); // Researcher
893        assert_eq!(tokens[2], Token::LBrace);
894        assert_eq!(tokens[3], Token::KwBelief);
895        assert_eq!(tokens[4], Token::Ident); // topic
896        assert_eq!(tokens[5], Token::Colon);
897        assert_eq!(tokens[6], Token::TyString);
898        assert_eq!(tokens[7], Token::KwOn);
899        assert_eq!(tokens[8], Token::KwStart);
900        assert_eq!(tokens[9], Token::LBrace);
901        assert_eq!(tokens[10], Token::KwLet);
902    }
903
904    #[test]
905    fn is_keyword_helper() {
906        assert!(Token::KwAgent.is_keyword());
907        assert!(Token::KwLet.is_keyword());
908        assert!(!Token::TyInt.is_keyword());
909        assert!(!Token::Ident.is_keyword());
910    }
911
912    #[test]
913    fn is_type_keyword_helper() {
914        assert!(Token::TyInt.is_type_keyword());
915        assert!(Token::TyAgent.is_type_keyword());
916        assert!(!Token::KwAgent.is_type_keyword());
917        assert!(!Token::Ident.is_type_keyword());
918    }
919
920    #[test]
921    fn is_literal_helper() {
922        assert!(Token::IntLit.is_literal());
923        assert!(Token::FloatLit.is_literal());
924        assert!(Token::StringLit.is_literal());
925        assert!(Token::KwTrue.is_literal());
926        assert!(Token::KwFalse.is_literal());
927        assert!(!Token::Ident.is_literal());
928    }
929
930    #[test]
931    fn is_operator_helper() {
932        assert!(Token::Plus.is_operator());
933        assert!(Token::EqEq.is_operator());
934        assert!(Token::PlusPlus.is_operator());
935        assert!(!Token::LBrace.is_operator());
936        assert!(!Token::Ident.is_operator());
937    }
938
939    #[test]
940    fn lex_module_keywords() {
941        let mut lexer = Token::lexer("mod use pub as super");
942        assert_eq!(lexer.next(), Some(Ok(Token::KwMod)));
943        assert_eq!(lexer.next(), Some(Ok(Token::KwUse)));
944        assert_eq!(lexer.next(), Some(Ok(Token::KwPub)));
945        assert_eq!(lexer.next(), Some(Ok(Token::KwAs)));
946        assert_eq!(lexer.next(), Some(Ok(Token::KwSuper)));
947        assert_eq!(lexer.next(), None);
948    }
949
950    #[test]
951    fn lex_path_separator() {
952        let mut lexer = Token::lexer("agents::Researcher");
953        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
954        assert_eq!(lexer.slice(), "agents");
955        assert_eq!(lexer.next(), Some(Ok(Token::ColonColon)));
956        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
957        assert_eq!(lexer.slice(), "Researcher");
958        assert_eq!(lexer.next(), None);
959    }
960
961    #[test]
962    fn lex_use_statement() {
963        let mut lexer = Token::lexer("use agents::{Researcher, Coordinator as Coord}");
964        assert_eq!(lexer.next(), Some(Ok(Token::KwUse)));
965        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // agents
966        assert_eq!(lexer.next(), Some(Ok(Token::ColonColon)));
967        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
968        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Researcher
969        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
970        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Coordinator
971        assert_eq!(lexer.next(), Some(Ok(Token::KwAs)));
972        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Coord
973        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
974        assert_eq!(lexer.next(), None);
975    }
976
977    #[test]
978    fn lex_pub_agent() {
979        let mut lexer = Token::lexer("pub agent Researcher");
980        assert_eq!(lexer.next(), Some(Ok(Token::KwPub)));
981        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
982        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
983        assert_eq!(lexer.next(), None);
984    }
985
986    #[test]
987    fn token_display() {
988        assert_eq!(format!("{}", Token::KwAgent), "agent");
989        assert_eq!(format!("{}", Token::TyInt), "Int");
990        assert_eq!(format!("{}", Token::IntLit), "<int>");
991        assert_eq!(format!("{}", Token::Ident), "<ident>");
992        assert_eq!(format!("{}", Token::LBrace), "{");
993        assert_eq!(format!("{}", Token::PlusPlus), "++");
994    }
995
996    #[test]
997    fn lex_type_keywords_record_enum_match_const() {
998        let mut lexer = Token::lexer("record enum match const");
999        assert_eq!(lexer.next(), Some(Ok(Token::KwRecord)));
1000        assert_eq!(lexer.next(), Some(Ok(Token::KwEnum)));
1001        assert_eq!(lexer.next(), Some(Ok(Token::KwMatch)));
1002        assert_eq!(lexer.next(), Some(Ok(Token::KwConst)));
1003        assert_eq!(lexer.next(), None);
1004    }
1005
1006    #[test]
1007    fn lex_fat_arrow() {
1008        let mut lexer = Token::lexer("=> -> =");
1009        assert_eq!(lexer.next(), Some(Ok(Token::FatArrow)));
1010        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1011        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
1012        assert_eq!(lexer.next(), None);
1013    }
1014
1015    #[test]
1016    fn lex_match_expression() {
1017        let mut lexer = Token::lexer("match status { Active => 1, Inactive => 0 }");
1018        assert_eq!(lexer.next(), Some(Ok(Token::KwMatch)));
1019        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // status
1020        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1021        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Active
1022        assert_eq!(lexer.next(), Some(Ok(Token::FatArrow)));
1023        assert_eq!(lexer.next(), Some(Ok(Token::IntLit))); // 1
1024        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
1025        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Inactive
1026        assert_eq!(lexer.next(), Some(Ok(Token::FatArrow)));
1027        assert_eq!(lexer.next(), Some(Ok(Token::IntLit))); // 0
1028        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1029        assert_eq!(lexer.next(), None);
1030    }
1031
1032    #[test]
1033    fn lex_record_declaration() {
1034        let mut lexer = Token::lexer("record Point { x: Int, y: Int }");
1035        assert_eq!(lexer.next(), Some(Ok(Token::KwRecord)));
1036        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Point
1037        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1038        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1039        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1040        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1041        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
1042        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // y
1043        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1044        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1045        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1046        assert_eq!(lexer.next(), None);
1047    }
1048
1049    #[test]
1050    fn lex_enum_declaration() {
1051        let mut lexer = Token::lexer("enum Status { Active, Pending, Done }");
1052        assert_eq!(lexer.next(), Some(Ok(Token::KwEnum)));
1053        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Status
1054        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1055        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Active
1056        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
1057        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Pending
1058        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
1059        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Done
1060        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1061        assert_eq!(lexer.next(), None);
1062    }
1063
1064    #[test]
1065    fn lex_const_declaration() {
1066        let mut lexer = Token::lexer("const MAX_RETRIES: Int = 3");
1067        assert_eq!(lexer.next(), Some(Ok(Token::KwConst)));
1068        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // MAX_RETRIES
1069        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1070        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1071        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
1072        assert_eq!(lexer.next(), Some(Ok(Token::IntLit))); // 3
1073        assert_eq!(lexer.next(), None);
1074    }
1075
1076    #[test]
1077    fn new_keywords_are_keywords() {
1078        assert!(Token::KwRecord.is_keyword());
1079        assert!(Token::KwEnum.is_keyword());
1080        assert!(Token::KwMatch.is_keyword());
1081        assert!(Token::KwConst.is_keyword());
1082    }
1083
1084    #[test]
1085    fn lex_loop_break() {
1086        let mut lexer = Token::lexer("loop { break }");
1087        assert_eq!(lexer.next(), Some(Ok(Token::KwLoop)));
1088        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1089        assert_eq!(lexer.next(), Some(Ok(Token::KwBreak)));
1090        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1091        assert_eq!(lexer.next(), None);
1092    }
1093
1094    #[test]
1095    fn lex_receives_receive() {
1096        let mut lexer = Token::lexer("agent Worker receives WorkerMsg { receive }");
1097        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
1098        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Worker
1099        assert_eq!(lexer.next(), Some(Ok(Token::KwReceives)));
1100        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // WorkerMsg
1101        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1102        assert_eq!(lexer.next(), Some(Ok(Token::KwReceive)));
1103        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1104        assert_eq!(lexer.next(), None);
1105    }
1106
1107    #[test]
1108    fn rfc6_keywords_are_keywords() {
1109        assert!(Token::KwLoop.is_keyword());
1110        assert!(Token::KwBreak.is_keyword());
1111        assert!(Token::KwReceives.is_keyword());
1112        assert!(Token::KwReceive.is_keyword());
1113    }
1114
1115    #[test]
1116    fn lex_error_handling_keywords() {
1117        let mut lexer = Token::lexer("fails try catch error");
1118        assert_eq!(lexer.next(), Some(Ok(Token::KwFails)));
1119        assert_eq!(lexer.next(), Some(Ok(Token::KwTry)));
1120        assert_eq!(lexer.next(), Some(Ok(Token::KwCatch)));
1121        assert_eq!(lexer.next(), Some(Ok(Token::KwError)));
1122        assert_eq!(lexer.next(), None);
1123    }
1124
1125    #[test]
1126    fn lex_try_catch_expression() {
1127        let mut lexer = Token::lexer("let x = try divine(prompt) catch { fallback }");
1128        assert_eq!(lexer.next(), Some(Ok(Token::KwLet)));
1129        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1130        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
1131        assert_eq!(lexer.next(), Some(Ok(Token::KwTry)));
1132        assert_eq!(lexer.next(), Some(Ok(Token::KwDivine)));
1133        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1134        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // prompt
1135        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1136        assert_eq!(lexer.next(), Some(Ok(Token::KwCatch)));
1137        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1138        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // fallback
1139        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1140        assert_eq!(lexer.next(), None);
1141    }
1142
1143    #[test]
1144    fn lex_fails_function() {
1145        let mut lexer = Token::lexer("fn fetch(url: String) -> String fails { }");
1146        assert_eq!(lexer.next(), Some(Ok(Token::KwFn)));
1147        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // fetch
1148        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1149        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // url
1150        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1151        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1152        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1153        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1154        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1155        assert_eq!(lexer.next(), Some(Ok(Token::KwFails)));
1156        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1157        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1158        assert_eq!(lexer.next(), None);
1159    }
1160
1161    #[test]
1162    fn lex_on_error_handler() {
1163        let mut lexer = Token::lexer("on error(e) { yield(fallback) }");
1164        assert_eq!(lexer.next(), Some(Ok(Token::KwOn)));
1165        assert_eq!(lexer.next(), Some(Ok(Token::KwError)));
1166        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1167        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // e
1168        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1169        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1170        assert_eq!(lexer.next(), Some(Ok(Token::KwYield)));
1171        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1172        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // fallback
1173        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1174        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1175        assert_eq!(lexer.next(), None);
1176    }
1177
1178    #[test]
1179    fn rfc7_keywords_are_keywords() {
1180        assert!(Token::KwFail.is_keyword());
1181        assert!(Token::KwFails.is_keyword());
1182        assert!(Token::KwTry.is_keyword());
1183        assert!(Token::KwCatch.is_keyword());
1184        assert!(Token::KwError.is_keyword());
1185    }
1186
1187    #[test]
1188    fn lex_fail_expression() {
1189        let mut lexer = Token::lexer("fail \"error message\"");
1190        assert_eq!(lexer.next(), Some(Ok(Token::KwFail)));
1191        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
1192        assert_eq!(lexer.next(), None);
1193    }
1194
1195    // =========================================================================
1196    // RFC-0009: Closures
1197    // =========================================================================
1198
1199    #[test]
1200    fn lex_closure_syntax() {
1201        // |x: Int| x + 1
1202        let mut lexer = Token::lexer("|x: Int| x + 1");
1203        assert_eq!(lexer.next(), Some(Ok(Token::Pipe)));
1204        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1205        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1206        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1207        assert_eq!(lexer.next(), Some(Ok(Token::Pipe)));
1208        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // x
1209        assert_eq!(lexer.next(), Some(Ok(Token::Plus)));
1210        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
1211        assert_eq!(lexer.next(), None);
1212    }
1213
1214    #[test]
1215    fn lex_empty_closure() {
1216        // || 42
1217        let mut lexer = Token::lexer("|| 42");
1218        assert_eq!(lexer.next(), Some(Ok(Token::Or))); // || lexes as Or
1219        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
1220        assert_eq!(lexer.next(), None);
1221    }
1222
1223    #[test]
1224    fn lex_fn_type() {
1225        // Fn(Int, String) -> Bool
1226        let mut lexer = Token::lexer("Fn(Int, String) -> Bool");
1227        assert_eq!(lexer.next(), Some(Ok(Token::TyFn)));
1228        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1229        assert_eq!(lexer.next(), Some(Ok(Token::TyInt)));
1230        assert_eq!(lexer.next(), Some(Ok(Token::Comma)));
1231        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1232        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1233        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1234        assert_eq!(lexer.next(), Some(Ok(Token::TyBool)));
1235        assert_eq!(lexer.next(), None);
1236    }
1237
1238    #[test]
1239    fn fn_is_type_keyword() {
1240        assert!(Token::TyFn.is_type_keyword());
1241    }
1242
1243    #[test]
1244    fn pipe_display() {
1245        assert_eq!(format!("{}", Token::Pipe), "|");
1246        assert_eq!(format!("{}", Token::TyFn), "Fn");
1247    }
1248
1249    // =========================================================================
1250    // RFC-0011: Tool Support
1251    // =========================================================================
1252
1253    #[test]
1254    fn lex_tool_keyword() {
1255        let mut lexer = Token::lexer("tool Http { fn get(url: String) -> String }");
1256        assert_eq!(lexer.next(), Some(Ok(Token::KwTool)));
1257        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Http
1258        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1259        assert_eq!(lexer.next(), Some(Ok(Token::KwFn)));
1260        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // get
1261        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1262        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // url
1263        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1264        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1265        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1266        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1267        assert_eq!(lexer.next(), Some(Ok(Token::TyString)));
1268        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1269        assert_eq!(lexer.next(), None);
1270    }
1271
1272    #[test]
1273    fn tool_is_keyword() {
1274        assert!(Token::KwTool.is_keyword());
1275    }
1276
1277    #[test]
1278    fn lex_agent_use_tool() {
1279        let mut lexer = Token::lexer("agent Fetcher { use Http }");
1280        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
1281        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Fetcher
1282        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1283        assert_eq!(lexer.next(), Some(Ok(Token::KwUse)));
1284        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Http
1285        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1286        assert_eq!(lexer.next(), None);
1287    }
1288
1289    // =========================================================================
1290    // Observability: trace and span
1291    // =========================================================================
1292
1293    #[test]
1294    fn lex_trace_span_keywords() {
1295        let mut lexer = Token::lexer("trace span");
1296        assert_eq!(lexer.next(), Some(Ok(Token::KwTrace)));
1297        assert_eq!(lexer.next(), Some(Ok(Token::KwSpan)));
1298        assert_eq!(lexer.next(), None);
1299    }
1300
1301    #[test]
1302    fn trace_span_are_keywords() {
1303        assert!(Token::KwTrace.is_keyword());
1304        assert!(Token::KwSpan.is_keyword());
1305    }
1306
1307    #[test]
1308    fn lex_span_block() {
1309        let mut lexer = Token::lexer("span \"fetch_data\" { let x = 1 }");
1310        assert_eq!(lexer.next(), Some(Ok(Token::KwSpan)));
1311        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
1312        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1313        assert_eq!(lexer.next(), Some(Ok(Token::KwLet)));
1314        assert_eq!(lexer.next(), Some(Ok(Token::Ident)));
1315        assert_eq!(lexer.next(), Some(Ok(Token::Eq)));
1316        assert_eq!(lexer.next(), Some(Ok(Token::IntLit)));
1317        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1318        assert_eq!(lexer.next(), None);
1319    }
1320
1321    // =========================================================================
1322    // Phase 3: Session types and algebraic effects
1323    // =========================================================================
1324
1325    #[test]
1326    fn lex_protocol_keywords() {
1327        let mut lexer = Token::lexer("protocol follows reply");
1328        assert_eq!(lexer.next(), Some(Ok(Token::KwProtocol)));
1329        assert_eq!(lexer.next(), Some(Ok(Token::KwFollows)));
1330        assert_eq!(lexer.next(), Some(Ok(Token::KwReply)));
1331        assert_eq!(lexer.next(), None);
1332    }
1333
1334    #[test]
1335    fn lex_effect_handler_keywords() {
1336        let mut lexer = Token::lexer("handler handles");
1337        assert_eq!(lexer.next(), Some(Ok(Token::KwHandler)));
1338        assert_eq!(lexer.next(), Some(Ok(Token::KwHandles)));
1339        assert_eq!(lexer.next(), None);
1340    }
1341
1342    #[test]
1343    fn protocol_keywords_are_keywords() {
1344        assert!(Token::KwProtocol.is_keyword());
1345        assert!(Token::KwFollows.is_keyword());
1346        assert!(Token::KwReply.is_keyword());
1347        assert!(Token::KwHandler.is_keyword());
1348        assert!(Token::KwHandles.is_keyword());
1349    }
1350
1351    #[test]
1352    fn lex_protocol_declaration() {
1353        let mut lexer = Token::lexer("protocol SchemaSync { Steward -> API: Changed }");
1354        assert_eq!(lexer.next(), Some(Ok(Token::KwProtocol)));
1355        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // SchemaSync
1356        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1357        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Steward
1358        assert_eq!(lexer.next(), Some(Ok(Token::Arrow)));
1359        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // API
1360        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1361        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Changed
1362        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1363        assert_eq!(lexer.next(), None);
1364    }
1365
1366    #[test]
1367    fn lex_agent_follows() {
1368        let mut lexer = Token::lexer("agent API follows SchemaSync as APISteward");
1369        assert_eq!(lexer.next(), Some(Ok(Token::KwAgent)));
1370        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // API
1371        assert_eq!(lexer.next(), Some(Ok(Token::KwFollows)));
1372        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // SchemaSync
1373        assert_eq!(lexer.next(), Some(Ok(Token::KwAs)));
1374        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // APISteward
1375        assert_eq!(lexer.next(), None);
1376    }
1377
1378    #[test]
1379    fn lex_handler_declaration() {
1380        let mut lexer = Token::lexer("handler DefaultLLM handles Infer { model: \"gpt-4\" }");
1381        assert_eq!(lexer.next(), Some(Ok(Token::KwHandler)));
1382        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // DefaultLLM
1383        assert_eq!(lexer.next(), Some(Ok(Token::KwHandles)));
1384        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Infer
1385        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1386        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // model
1387        assert_eq!(lexer.next(), Some(Ok(Token::Colon)));
1388        assert_eq!(lexer.next(), Some(Ok(Token::StringLit)));
1389        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1390        assert_eq!(lexer.next(), None);
1391    }
1392
1393    #[test]
1394    fn lex_reply_expression() {
1395        let mut lexer = Token::lexer("reply(Ack {})");
1396        assert_eq!(lexer.next(), Some(Ok(Token::KwReply)));
1397        assert_eq!(lexer.next(), Some(Ok(Token::LParen)));
1398        assert_eq!(lexer.next(), Some(Ok(Token::Ident))); // Ack
1399        assert_eq!(lexer.next(), Some(Ok(Token::LBrace)));
1400        assert_eq!(lexer.next(), Some(Ok(Token::RBrace)));
1401        assert_eq!(lexer.next(), Some(Ok(Token::RParen)));
1402        assert_eq!(lexer.next(), None);
1403    }
1404
1405    #[test]
1406    fn token_display_phase3() {
1407        assert_eq!(format!("{}", Token::KwProtocol), "protocol");
1408        assert_eq!(format!("{}", Token::KwFollows), "follows");
1409        assert_eq!(format!("{}", Token::KwReply), "reply");
1410        assert_eq!(format!("{}", Token::KwHandler), "handler");
1411        assert_eq!(format!("{}", Token::KwHandles), "handles");
1412    }
1413}
sage_parser/token.rs

sage_parser/
token.rs
