lexer/
token.rs

1use serde::{Deserialize, Serialize};
2use std::fmt;
3use std::fmt::Formatter;
4
5#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
6pub struct Token {
7    pub kind: TokenKind,
8    pub span: Span,
9}
10
11#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
12pub struct Span {
13    pub start: usize,
14    pub end: usize,
15}
16
17impl fmt::Display for Token {
18    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
19        write!(f, "start: {}, end: {}, kind: {}", self.span.start, self.span.end, self.kind)
20    }
21}
22
23#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
24#[serde(tag = "type", content = "value")]
25pub enum TokenKind {
26    ILLEGAL,
27    EOF,
28
29    // Identifiers + literals
30    IDENTIFIER { name: String },
31    INT(i64),
32    STRING(String),
33
34    // Operators
35    ASSIGN,   // =
36    PLUS,     // +
37    MINUS,    // -
38    BANG,     // !
39    ASTERISK, // *
40    SLASH,    // /
41
42    LT, // <
43    GT, // >
44
45    EQ,    // ==
46    NotEq, // !=
47
48    // delimiters
49    COMMA,
50    SEMICOLON,
51    COLON,
52
53    LPAREN,
54    RPAREN,
55    LBRACE,   // {
56    RBRACE,   // }
57    LBRACKET, // [
58    RBRACKET, // ]
59
60    // keywords
61    FUNCTION,
62    LET,
63    TRUE,
64    FALSE,
65    IF,
66    ELSE,
67    RETURN,
68}
69
70pub fn lookup_identifier(identifier: &str) -> TokenKind {
71    match identifier {
72        "fn" => TokenKind::FUNCTION,
73        "let" => TokenKind::LET,
74        "true" => TokenKind::TRUE,
75        "false" => TokenKind::FALSE,
76        "if" => TokenKind::IF,
77        "else" => TokenKind::ELSE,
78        "return" => TokenKind::RETURN,
79        _ => TokenKind::IDENTIFIER {
80            name: identifier.to_string(),
81        },
82    }
83}
84
85impl fmt::Display for TokenKind {
86    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
87        match self {
88            TokenKind::IDENTIFIER {
89                name,
90            } => write!(f, "{}", name),
91            TokenKind::INT(i) => write!(f, "{}", i),
92            TokenKind::STRING(s) => write!(f, "{}", s),
93            TokenKind::ASSIGN => write!(f, "="),
94            TokenKind::PLUS => write!(f, "+"),
95            TokenKind::MINUS => write!(f, "-"),
96            TokenKind::BANG => write!(f, "!"),
97            TokenKind::ASTERISK => write!(f, "*"),
98            TokenKind::SLASH => write!(f, "/"),
99            TokenKind::LT => write!(f, "<"),
100            TokenKind::GT => write!(f, ">"),
101            TokenKind::EQ => write!(f, "=="),
102            TokenKind::NotEq => write!(f, "!="),
103            TokenKind::COMMA => write!(f, ","),
104            TokenKind::SEMICOLON => write!(f, ";"),
105            TokenKind::LPAREN => write!(f, "("),
106            TokenKind::RPAREN => write!(f, ")"),
107            TokenKind::LBRACE => write!(f, "{{"),
108            TokenKind::RBRACE => write!(f, "}}"),
109            TokenKind::LBRACKET => write!(f, "["),
110            TokenKind::RBRACKET => write!(f, "]"),
111            TokenKind::FUNCTION => write!(f, "fn"),
112            TokenKind::LET => write!(f, "let"),
113            TokenKind::TRUE => write!(f, "true"),
114            TokenKind::FALSE => write!(f, "false"),
115            TokenKind::IF => write!(f, "if"),
116            TokenKind::ELSE => write!(f, "else"),
117            TokenKind::RETURN => write!(f, "return"),
118            TokenKind::ILLEGAL => write!(f, "ILLEGAL"),
119            TokenKind::EOF => write!(f, "EOF"),
120            TokenKind::COLON => write!(f, ":"),
121        }
122    }
123}
lexer/token.rs

lexer/
token.rs
