lexer/
token.rs

1use serde::{Deserialize, Serialize};
2use std::fmt;
3use std::fmt::Formatter;
4
5#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
6pub struct Token {
7    pub kind: TokenKind,
8    pub span: Span,
9}
10
11#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
12pub struct Span {
13    pub start: usize,
14    pub end: usize,
15}
16
17impl fmt::Display for Token {
18    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
19        write!(f, "start: {}, end: {}, kind: {}", self.span.start, self.span.end, self.kind)
20    }
21}
22
23#[derive(Clone, Debug, Eq, Hash, Ord, Serialize, Deserialize, PartialOrd, PartialEq)]
24#[serde(tag = "type", content = "value")]
25pub enum TokenKind {
26    ILLEGAL,
27    EOF,
28
29    // Identifiers + literals
30    IDENTIFIER { name: String },
31    INT(i64),
32    STRING(String),
33
34    // Operators
35    ASSIGN,   // =
36    PLUS,     // +
37    MINUS,    // -
38    BANG,     // !
39    ASTERISK, // *
40    SLASH,    // /
41
42    LT, // <
43    GT, // >
44
45    EQ,    // ==
46    NotEq, // !=
47
48    // delimiters
49    COMMA,
50    SEMICOLON,
51    COLON,
52    DOT,
53    QUESTION, // ? — only meaningful in type position (`T?`)
54
55    LPAREN,
56    RPAREN,
57    LBRACE,   // {
58    RBRACE,   // }
59    LBRACKET, // [
60    RBRACKET, // ]
61
62    // keywords
63    FUNCTION,
64    LET,
65    TRUE,
66    FALSE,
67    IF,
68    ELSE,
69    RETURN,
70    CLASS,
71    THIS,
72    NEW,
73    DEBUGGER,
74}
75
76pub fn lookup_identifier(identifier: &str) -> TokenKind {
77    match identifier {
78        "fn" => TokenKind::FUNCTION,
79        "let" => TokenKind::LET,
80        "true" => TokenKind::TRUE,
81        "false" => TokenKind::FALSE,
82        "if" => TokenKind::IF,
83        "else" => TokenKind::ELSE,
84        "return" => TokenKind::RETURN,
85        "class" => TokenKind::CLASS,
86        "this" => TokenKind::THIS,
87        "new" => TokenKind::NEW,
88        "debugger" => TokenKind::DEBUGGER,
89        _ => TokenKind::IDENTIFIER {
90            name: identifier.to_string(),
91        },
92    }
93}
94
95impl fmt::Display for TokenKind {
96    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
97        match self {
98            TokenKind::IDENTIFIER {
99                name,
100            } => write!(f, "{}", name),
101            TokenKind::INT(i) => write!(f, "{}", i),
102            TokenKind::STRING(s) => write!(f, "{}", s),
103            TokenKind::ASSIGN => write!(f, "="),
104            TokenKind::PLUS => write!(f, "+"),
105            TokenKind::MINUS => write!(f, "-"),
106            TokenKind::BANG => write!(f, "!"),
107            TokenKind::ASTERISK => write!(f, "*"),
108            TokenKind::SLASH => write!(f, "/"),
109            TokenKind::LT => write!(f, "<"),
110            TokenKind::GT => write!(f, ">"),
111            TokenKind::EQ => write!(f, "=="),
112            TokenKind::NotEq => write!(f, "!="),
113            TokenKind::COMMA => write!(f, ","),
114            TokenKind::SEMICOLON => write!(f, ";"),
115            TokenKind::LPAREN => write!(f, "("),
116            TokenKind::RPAREN => write!(f, ")"),
117            TokenKind::LBRACE => write!(f, "{{"),
118            TokenKind::RBRACE => write!(f, "}}"),
119            TokenKind::LBRACKET => write!(f, "["),
120            TokenKind::RBRACKET => write!(f, "]"),
121            TokenKind::FUNCTION => write!(f, "fn"),
122            TokenKind::LET => write!(f, "let"),
123            TokenKind::TRUE => write!(f, "true"),
124            TokenKind::FALSE => write!(f, "false"),
125            TokenKind::IF => write!(f, "if"),
126            TokenKind::ELSE => write!(f, "else"),
127            TokenKind::RETURN => write!(f, "return"),
128            TokenKind::ILLEGAL => write!(f, "ILLEGAL"),
129            TokenKind::EOF => write!(f, "EOF"),
130            TokenKind::COLON => write!(f, ":"),
131            TokenKind::DOT => write!(f, "."),
132            TokenKind::QUESTION => write!(f, "?"),
133            TokenKind::CLASS => write!(f, "class"),
134            TokenKind::THIS => write!(f, "this"),
135            TokenKind::NEW => write!(f, "new"),
136            TokenKind::DEBUGGER => write!(f, "debugger"),
137        }
138    }
139}
lexer/token.rs

lexer/
token.rs
