lexer/
token.rs

1use serde::{Deserialize, Serialize};
2use std::fmt;
3use std::fmt::Formatter;
4
5#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
6pub struct Token {
7    pub kind: TokenKind,
8    pub span: Span,
9}
10
11#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
12pub struct Span {
13    pub start: usize,
14    pub end: usize,
15}
16
17impl fmt::Display for Token {
18    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
19        write!(f, "start: {}, end: {}, kind: {}", self.span.start, self.span.end, self.kind)
20    }
21}
22
23#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
24#[serde(tag = "type", content = "value")]
25pub enum TokenKind {
26    ILLEGAL,
27    EOF,
28
29    // Identifiers + literals
30    IDENTIFIER { name: String },
31    INT(i64),
32    STRING(String),
33
34    // Operators
35    ASSIGN,   // =
36    PLUS,     // +
37    MINUS,    // -
38    BANG,     // !
39    ASTERISK, // *
40    SLASH,    // /
41
42    LT, // <
43    GT, // >
44
45    EQ,    // ==
46    NotEq, // !=
47
48    // delimiters
49    COMMA,
50    SEMICOLON,
51    COLON,
52
53    LPAREN,
54    RPAREN,
55    LBRACE,   // {
56    RBRACE,   // }
57    LBRACKET, // [
58    RBRACKET, // ]
59
60    // keywords
61    FUNCTION,
62    LET,
63    TRUE,
64    FALSE,
65    IF,
66    ELSE,
67    RETURN,
68}
69
70pub fn lookup_identifier(identifier: &str) -> TokenKind {
71    match identifier {
72        "fn" => TokenKind::FUNCTION,
73        "let" => TokenKind::LET,
74        "true" => TokenKind::TRUE,
75        "false" => TokenKind::FALSE,
76        "if" => TokenKind::IF,
77        "else" => TokenKind::ELSE,
78        "return" => TokenKind::RETURN,
79        _ => TokenKind::IDENTIFIER { name: identifier.to_string() },
80    }
81}
82
83impl fmt::Display for TokenKind {
84    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
85        match self {
86            TokenKind::IDENTIFIER { name } => write!(f, "{}", name),
87            TokenKind::INT(i) => write!(f, "{}", i),
88            TokenKind::STRING(s) => write!(f, "{}", s),
89            TokenKind::ASSIGN => write!(f, "="),
90            TokenKind::PLUS => write!(f, "+"),
91            TokenKind::MINUS => write!(f, "-"),
92            TokenKind::BANG => write!(f, "!"),
93            TokenKind::ASTERISK => write!(f, "*"),
94            TokenKind::SLASH => write!(f, "/"),
95            TokenKind::LT => write!(f, "<"),
96            TokenKind::GT => write!(f, ">"),
97            TokenKind::EQ => write!(f, "=="),
98            TokenKind::NotEq => write!(f, "!="),
99            TokenKind::COMMA => write!(f, ","),
100            TokenKind::SEMICOLON => write!(f, ";"),
101            TokenKind::LPAREN => write!(f, "("),
102            TokenKind::RPAREN => write!(f, ")"),
103            TokenKind::LBRACE => write!(f, "{{"),
104            TokenKind::RBRACE => write!(f, "}}"),
105            TokenKind::LBRACKET => write!(f, "["),
106            TokenKind::RBRACKET => write!(f, "]"),
107            TokenKind::FUNCTION => write!(f, "fn"),
108            TokenKind::LET => write!(f, "let"),
109            TokenKind::TRUE => write!(f, "true"),
110            TokenKind::FALSE => write!(f, "false"),
111            TokenKind::IF => write!(f, "if"),
112            TokenKind::ELSE => write!(f, "else"),
113            TokenKind::RETURN => write!(f, "return"),
114            TokenKind::ILLEGAL => write!(f, "ILLEGAL"),
115            TokenKind::EOF => write!(f, "EOF"),
116            TokenKind::COLON => write!(f, ":"),
117        }
118    }
119}
lexer/token.rs

lexer/
token.rs
