lexer/
token.rs

1use serde::{Deserialize, Serialize};
2use std::fmt;
3use std::fmt::Formatter;
4
5#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
6pub struct Token {
7    pub kind: TokenKind,
8    pub span: Span,
9}
10
11#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
12pub struct Span {
13    pub start: usize,
14    pub end: usize,
15}
16
17impl fmt::Display for Token {
18    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
19        write!(f, "start: {}, end: {}, kind: {}", self.span.start, self.span.end, self.kind)
20    }
21}
22
23#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
24#[serde(tag = "type", content = "value")]
25pub enum TokenKind {
26    ILLEGAL,
27    EOF,
28
29    // Identifiers + literals
30    IDENTIFIER { name: String },
31    INT(i64),
32    STRING(String),
33
34    // Operators
35    ASSIGN,   // =
36    PLUS,     // +
37    MINUS,    // -
38    BANG,     // !
39    ASTERISK, // *
40    SLASH,    // /
41
42    LT, // <
43    GT, // >
44
45    EQ,    // ==
46    NotEq, // !=
47
48    // delimiters
49    COMMA,
50    SEMICOLON,
51    COLON,
52    DOT,
53
54    LPAREN,
55    RPAREN,
56    LBRACE,   // {
57    RBRACE,   // }
58    LBRACKET, // [
59    RBRACKET, // ]
60
61    // keywords
62    FUNCTION,
63    LET,
64    TRUE,
65    FALSE,
66    IF,
67    ELSE,
68    RETURN,
69    CLASS,
70    THIS,
71    NEW,
72}
73
74pub fn lookup_identifier(identifier: &str) -> TokenKind {
75    match identifier {
76        "fn" => TokenKind::FUNCTION,
77        "let" => TokenKind::LET,
78        "true" => TokenKind::TRUE,
79        "false" => TokenKind::FALSE,
80        "if" => TokenKind::IF,
81        "else" => TokenKind::ELSE,
82        "return" => TokenKind::RETURN,
83        "class" => TokenKind::CLASS,
84        "this" => TokenKind::THIS,
85        "new" => TokenKind::NEW,
86        _ => TokenKind::IDENTIFIER {
87            name: identifier.to_string(),
88        },
89    }
90}
91
92impl fmt::Display for TokenKind {
93    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
94        match self {
95            TokenKind::IDENTIFIER {
96                name,
97            } => write!(f, "{}", name),
98            TokenKind::INT(i) => write!(f, "{}", i),
99            TokenKind::STRING(s) => write!(f, "{}", s),
100            TokenKind::ASSIGN => write!(f, "="),
101            TokenKind::PLUS => write!(f, "+"),
102            TokenKind::MINUS => write!(f, "-"),
103            TokenKind::BANG => write!(f, "!"),
104            TokenKind::ASTERISK => write!(f, "*"),
105            TokenKind::SLASH => write!(f, "/"),
106            TokenKind::LT => write!(f, "<"),
107            TokenKind::GT => write!(f, ">"),
108            TokenKind::EQ => write!(f, "=="),
109            TokenKind::NotEq => write!(f, "!="),
110            TokenKind::COMMA => write!(f, ","),
111            TokenKind::SEMICOLON => write!(f, ";"),
112            TokenKind::LPAREN => write!(f, "("),
113            TokenKind::RPAREN => write!(f, ")"),
114            TokenKind::LBRACE => write!(f, "{{"),
115            TokenKind::RBRACE => write!(f, "}}"),
116            TokenKind::LBRACKET => write!(f, "["),
117            TokenKind::RBRACKET => write!(f, "]"),
118            TokenKind::FUNCTION => write!(f, "fn"),
119            TokenKind::LET => write!(f, "let"),
120            TokenKind::TRUE => write!(f, "true"),
121            TokenKind::FALSE => write!(f, "false"),
122            TokenKind::IF => write!(f, "if"),
123            TokenKind::ELSE => write!(f, "else"),
124            TokenKind::RETURN => write!(f, "return"),
125            TokenKind::ILLEGAL => write!(f, "ILLEGAL"),
126            TokenKind::EOF => write!(f, "EOF"),
127            TokenKind::COLON => write!(f, ":"),
128            TokenKind::DOT => write!(f, "."),
129            TokenKind::CLASS => write!(f, "class"),
130            TokenKind::THIS => write!(f, "this"),
131            TokenKind::NEW => write!(f, "new"),
132        }
133    }
134}
lexer/token.rs

lexer/
token.rs
