roan_ast/lexer/
token.rs

1use roan_error::span::TextSpan;
2use std::{
3    fmt,
4    fmt::{Debug, Display, Formatter},
5};
6
7#[derive(Clone, Debug, PartialEq)]
8pub struct Token {
9    pub kind: TokenKind,
10    pub span: TextSpan,
11}
12
13impl Token {
14    /// Creates a new `Token` with the specified `TokenKind` and `TextSpan`.
15    pub fn new(kind: TokenKind, span: TextSpan) -> Self {
16        Self { kind, span }
17    }
18
19    /// Returns the literal value of the token.
20    pub fn literal(&self) -> String {
21        self.span.literal.clone()
22    }
23
24    /// Checks if the token is a string.
25    pub fn is_string(&self) -> bool {
26        matches!(self.kind, TokenKind::String(_))
27    }
28
29    /// Tries to convert the token to a boolean.
30    ///
31    /// Throws a panic if the token is not a boolean.
32    pub fn as_bool(&self) -> Option<bool> {
33        match self.kind {
34            TokenKind::True => Some(true),
35            TokenKind::False => Some(false),
36            _ => unreachable!("Token is not a boolean"),
37        }
38    }
39}
40
41impl Display for TokenKind {
42    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
43        match self {
44            // Separators
45            TokenKind::LeftParen => write!(f, "("),
46            TokenKind::RightParen => write!(f, ")"),
47            TokenKind::LeftBrace => write!(f, "{{"),
48            TokenKind::RightBrace => write!(f, "}}"),
49            TokenKind::LeftBracket => write!(f, "["),
50            TokenKind::RightBracket => write!(f, "]"),
51            TokenKind::Comma => write!(f, ","),
52            TokenKind::Dot => write!(f, "."),
53            TokenKind::Colon => write!(f, ":"),
54            TokenKind::Semicolon => write!(f, ";"),
55            TokenKind::Arrow => write!(f, "->"),
56            TokenKind::DoubleDot => write!(f, ".."),
57            TokenKind::TripleDot => write!(f, "..."),
58            TokenKind::DoubleColon => write!(f, "::"),
59
60            // Literals
61            TokenKind::Identifier => write!(f, "Identifier"),
62            TokenKind::String(s) => write!(f, "{}", s),
63            TokenKind::Float(r) => write!(f, "{}", r),
64            TokenKind::Integer(i) => write!(f, "{}", i),
65            TokenKind::Char(c) => write!(f, "{}", c),
66
67            // Keywords
68            TokenKind::Fn => write!(f, "fn"),
69            TokenKind::Let => write!(f, "let"),
70            TokenKind::If => write!(f, "if"),
71            TokenKind::Else => write!(f, "else"),
72            TokenKind::While => write!(f, "while"),
73            TokenKind::For => write!(f, "for"),
74            TokenKind::In => write!(f, "in"),
75            TokenKind::Return => write!(f, "return"),
76            TokenKind::Break => write!(f, "break"),
77            TokenKind::Continue => write!(f, "continue"),
78            TokenKind::Use => write!(f, "use"),
79            TokenKind::Pub => write!(f, "pub"),
80            TokenKind::From => write!(f, "from"),
81            TokenKind::Throw => write!(f, "throw"),
82            TokenKind::Try => write!(f, "try"),
83            TokenKind::Catch => write!(f, "catch"),
84            TokenKind::Loop => write!(f, "loop"),
85            TokenKind::True => write!(f, "true"),
86            TokenKind::False => write!(f, "false"),
87            TokenKind::Null => write!(f, "null"),
88            TokenKind::Impl => write!(f, "impl"),
89            TokenKind::Struct => write!(f, "struct"),
90            TokenKind::Trait => write!(f, "trait"),
91            TokenKind::Then => write!(f, "then"),
92            TokenKind::Const => write!(f, "const"),
93
94            // Operators
95            TokenKind::Plus => write!(f, "+"),
96            TokenKind::Minus => write!(f, "-"),
97            TokenKind::Asterisk => write!(f, "*"),
98            TokenKind::Slash => write!(f, "/"),
99            TokenKind::Equals => write!(f, "="),
100            TokenKind::Ampersand => write!(f, "&"),
101            TokenKind::Pipe => write!(f, "|"),
102            TokenKind::Caret => write!(f, "^"),
103            TokenKind::DoubleAsterisk => write!(f, "**"),
104            TokenKind::Percent => write!(f, "%"),
105            TokenKind::Tilde => write!(f, "~"),
106            TokenKind::GreaterThan => write!(f, ">"),
107            TokenKind::LessThan => write!(f, "<"),
108            TokenKind::GreaterThanEquals => write!(f, ">="),
109            TokenKind::LessThanEquals => write!(f, "<="),
110            TokenKind::EqualsEquals => write!(f, "=="),
111            TokenKind::BangEquals => write!(f, "!="),
112            TokenKind::Bang => write!(f, "!"),
113            TokenKind::And => write!(f, "&&"),
114            TokenKind::Or => write!(f, "||"),
115            TokenKind::Increment => write!(f, "++"),
116            TokenKind::Decrement => write!(f, "--"),
117            TokenKind::MinusEquals => write!(f, "-="),
118            TokenKind::PlusEquals => write!(f, "+="),
119            TokenKind::MultiplyEquals => write!(f, "*="),
120            TokenKind::DivideEquals => write!(f, "/="),
121            TokenKind::DoubleLessThan => write!(f, "<<"),
122            TokenKind::DoubleGreaterThan => write!(f, ">>"),
123            TokenKind::QuestionMark => write!(f, "?"),
124
125            // Others
126            TokenKind::EOF => write!(f, "EOF"),
127            TokenKind::Whitespace => write!(f, "Whitespace"),
128            TokenKind::Bad => write!(f, "Bad"),
129            TokenKind::Comment => write!(f, "Comment"),
130        }
131    }
132}
133
134#[derive(Debug, Clone, PartialEq)]
135pub enum TokenKind {
136    // Separators
137    LeftParen,
138    RightParen,
139    LeftBrace,
140    RightBrace,
141    LeftBracket,
142    RightBracket,
143    Comma,
144    Dot,
145    Colon,
146    Semicolon,
147    Arrow,
148    DoubleDot,
149    TripleDot,
150    DoubleColon,
151
152    // Literals
153    Identifier,
154    String(String),
155    Float(f64),
156    Integer(i64),
157    Char(char),
158
159    // Keywords
160    Fn,
161    Let,
162    If,
163    Else,
164    While,
165    For,
166    In,
167    Return,
168    Break,
169    Continue,
170    Use,
171    Pub,
172    From,
173    Throw,
174    Try,
175    Catch,
176    Loop,
177    True,
178    False,
179    Null,
180    Impl,
181    Struct,
182    Trait,
183    Then,
184    Const,
185
186    // Operators
187    Plus,              // +
188    Minus,             // -
189    Asterisk,          // *
190    Slash,             // /
191    Equals,            // =
192    Ampersand,         // &
193    Pipe,              // |
194    Caret,             // ^
195    DoubleAsterisk,    // **
196    Percent,           // %
197    Tilde,             // ~
198    GreaterThan,       // >
199    LessThan,          // <
200    GreaterThanEquals, // >=
201    LessThanEquals,    // <=
202    EqualsEquals,      // ==
203    BangEquals,        // !=
204    Bang,              // !
205    And,               // &&
206    Or,                // ||
207    Increment,         // ++
208    Decrement,         // --
209    MinusEquals,       // -=
210    PlusEquals,        // +=
211    MultiplyEquals,    // *=
212    DivideEquals,      // /=
213    DoubleLessThan,    // <<,
214    DoubleGreaterThan, // >>,
215    QuestionMark,      // ?
216
217    EOF,
218    Whitespace,
219    Bad,
220    Comment,
221}
222
223impl TokenKind {
224    pub fn is_keyword(&self) -> bool {
225        matches!(
226            self,
227            TokenKind::Fn
228                | TokenKind::Let
229                | TokenKind::If
230                | TokenKind::Else
231                | TokenKind::While
232                | TokenKind::For
233                | TokenKind::In
234                | TokenKind::Return
235                | TokenKind::Break
236                | TokenKind::Continue
237                | TokenKind::Use
238                | TokenKind::Pub
239                | TokenKind::From
240                | TokenKind::Throw
241                | TokenKind::Try
242                | TokenKind::Catch
243                | TokenKind::Loop
244                | TokenKind::True
245                | TokenKind::False
246                | TokenKind::Null
247                | TokenKind::Impl
248                | TokenKind::Struct
249                | TokenKind::Trait
250                | TokenKind::Then
251                | TokenKind::Const
252        )
253    }
254    
255    pub fn is_operator(&self) -> bool {
256        matches!(
257            self,
258            TokenKind::Plus
259                | TokenKind::Minus
260                | TokenKind::Asterisk
261                | TokenKind::Slash
262                | TokenKind::Equals
263                | TokenKind::Ampersand
264                | TokenKind::Pipe
265                | TokenKind::Caret
266                | TokenKind::DoubleAsterisk
267                | TokenKind::Percent
268                | TokenKind::Tilde
269                | TokenKind::GreaterThan
270                | TokenKind::LessThan
271                | TokenKind::GreaterThanEquals
272                | TokenKind::LessThanEquals
273                | TokenKind::EqualsEquals
274                | TokenKind::BangEquals
275                | TokenKind::Bang
276                | TokenKind::And
277                | TokenKind::Or
278                | TokenKind::Increment
279                | TokenKind::Decrement
280                | TokenKind::MinusEquals
281                | TokenKind::PlusEquals
282                | TokenKind::MultiplyEquals
283                | TokenKind::DivideEquals
284                | TokenKind::DoubleLessThan
285                | TokenKind::DoubleGreaterThan
286                | TokenKind::QuestionMark
287        )
288    }
289    
290    pub fn is_separator(&self) -> bool {
291        matches!(
292            self,
293            TokenKind::LeftParen
294                | TokenKind::RightParen
295                | TokenKind::LeftBrace
296                | TokenKind::RightBrace
297                | TokenKind::LeftBracket
298                | TokenKind::RightBracket
299                | TokenKind::Comma
300                | TokenKind::Dot
301                | TokenKind::Colon
302                | TokenKind::Semicolon
303                | TokenKind::Arrow
304                | TokenKind::DoubleDot
305                | TokenKind::TripleDot
306                | TokenKind::DoubleColon
307        )
308    }
309}
roan_ast/lexer/token.rs

roan_ast/lexer/
token.rs
