monkey_rs/
token.rs

1/*!
2# Token
3
4`token` defines the tokens accepted from a Monkey source file.
5*/
6use std::fmt;
7
8/// Defines the tokens in the token stream generated by the lexer.
9#[derive(Debug, Clone, PartialEq, Eq)]
10pub enum Token {
11    /* Reserved */
12    /// An illegal, unrecognized token.
13    Illegal,
14    /// Token marking the end of a file stream.
15    Eof,
16
17    /* Identifiers and literals */
18    /// Identifier, e.g., `add`, `foobar`, `x`, `y`, ...
19    Ident(String),
20    /// Integer literal, e.g., `[0-9]`
21    Int(i32),
22    /// A string literal, e.g., \"Hello, world!\"
23    String(String),
24
25    /* Operators */
26    /// Assignment operator `=`
27    Assign,
28    /// Summation arithmetic operator `+`
29    Plus,
30    /// Subtraction arithmetic operator `-`
31    Minus,
32    /// Negation logical operator `!`
33    Bang,
34    /// Multiplication arithmetic operator `*`
35    Asterisk,
36    /// Division arithmetic operator `/`
37    Slash,
38    /// Less than logical operator `<`
39    Lt,
40    /// Greater than logical operator `>`
41    Gt,
42    /// Equality logical operator `==`
43    Eq,
44    /// Inequality logical operator `!=`
45    NotEq,
46
47    /* Delimiters */
48    /// Comma `,`
49    Comma,
50    /// Semicolon `;`
51    Semicolon,
52    /// Colon `:`
53    Colon,
54
55    /// Left parenthesis `(`
56    LParen,
57    /// Right parenthesis `)`
58    RParen,
59
60    /// Left curly brace `{{`
61    LBrace,
62    /// Right curly bracket `}}`
63    RBrace,
64
65    /// Left square bracket `[`
66    LBracket,
67    /// Right square bracket `]`
68    RBracket,
69
70    /* Keywords */
71    /// `fn` keyword
72    Function,
73    /// `let` keyword
74    Let,
75    /// `true` keyword
76    True,
77    /// `false` keyword
78    False,
79    /// `if` keyword
80    If,
81    /// `else` keyword
82    Else,
83    /// `return` keyword
84    Return,
85}
86
87impl fmt::Display for Token {
88    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
89        match self {
90            Token::Illegal => write!(f, "ILLEGAL"),
91            Token::Eof => write!(f, "EOF"),
92            Token::Ident(id) => write!(f, "{}", id),
93            Token::Int(i) => write!(f, "{}", i),
94            Token::Assign => write!(f, "="),
95            Token::Plus => write!(f, "+"),
96            Token::Minus => write!(f, "-"),
97            Token::Bang => write!(f, "!"),
98            Token::Asterisk => write!(f, "*"),
99            Token::Slash => write!(f, "/"),
100            Token::Lt => write!(f, "<"),
101            Token::Gt => write!(f, ">"),
102            Token::Eq => write!(f, "=="),
103            Token::NotEq => write!(f, "!="),
104            Token::Comma => write!(f, ","),
105            Token::Semicolon => write!(f, ";"),
106            Token::LParen => write!(f, "("),
107            Token::RParen => write!(f, ")"),
108            Token::LBrace => write!(f, "{{"),
109            Token::RBrace => write!(f, "}}"),
110            Token::Function => write!(f, "FUNCTION"),
111            Token::Let => write!(f, "LET"),
112            Token::True => write!(f, "TRUE"),
113            Token::False => write!(f, "FALSE"),
114            Token::If => write!(f, "IF"),
115            Token::Else => write!(f, "ELSE"),
116            Token::Return => write!(f, "RETURN"),
117            Token::String(str) => write!(f, "{}", str),
118            Token::LBracket => write!(f, "["),
119            Token::RBracket => write!(f, "]"),
120            Token::Colon => write!(f, ":"),
121        }
122    }
123}
124
125/// Return the token associated with a raw identifier. If the identifier is not
126/// associated with a defined keyword, (e.g., a user-defined identifier),
127/// defaults to `Token::Ident`.
128pub fn lookup_ident(ident: &str) -> Token {
129    match ident {
130        /* keywords */
131        "fn" => Token::Function,
132        "let" => Token::Let,
133        "true" => Token::True,
134        "false" => Token::False,
135        "if" => Token::If,
136        "else" => Token::Else,
137        "return" => Token::Return,
138
139        /* user-defined identifier */
140        _ => Token::Ident(ident.to_string()),
141    }
142}
monkey_rs/token.rs

monkey_rs/
token.rs
