1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
/// A type-alias that represents the start and end point of a token.
pub type Span = (usize, usize);

/// Represents the "kind" of a token.
/// 
/// They are separated into groups in the source code, where each group signifies a sub-type of token.
#[derive(Debug, PartialEq, Clone)]
pub enum TokenKind {
    Fn,
    Let,
    If,
    Else,
    While,
    Return,
    Break,
    Continue,

    True,
    False,

    Identifier(String),
    String(String),
    Number(f64),

    Colon,
    DoubleColon,
    SemiColon,
    Comma,

    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    LeftBracket,
    RightBracket,

    Plus,
    Minus,
    Asterisk,
    Slash,
    Percent,
    DoubleAsterisk,

    Equals,

    Eof,
}

/// Stores information regarding a token.
/// 
/// The `Token` type holds information about the type of a token (`TokenType`), as well as it's `line` and `span` (start and end column) in the source code.
#[derive(Debug, Clone)]
pub struct Token {
    pub kind: TokenKind,
    pub line: usize,
    pub span: Span,
}

impl Token {
    pub fn new(kind: TokenKind, line: usize, span: Span) -> Self {
        Self { kind, line, span }
    }

    pub fn eof() -> Self {
        Self {
            kind: TokenKind::Eof,
            line: 0,
            span: (0, 0),
        }
    }
}

impl Default for Token {
    fn default() -> Self {
        Self {
            kind: TokenKind::Eof,
            line: 0,
            span: (0, 0),
        }
    }
}