1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

use std::fmt::Display;
use crate::common::data::Data;

// TODO: remove associated data from tokens.

/// These are the different tokens the lexer will output.
/// `Token`s with data contain that data,
/// e.g. a boolean will be a `Data::Boolean(...)`, not just a string.
/// `Token`s can be spanned using `Spanned<Token>`.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum Token {
    // Delimiters
    OpenBracket,
    CloseBracket,
    OpenParen,
    CloseParen,
    Sep,
    Pair,

    // Keywords
    Syntax,
    Assign,
    Lambda,
    Compose,
    Magic,
    // pseudokeywords
    Keyword(String),

    // Datatypes
    // TODO: just have one variant, `Data`
    Unit,
    Number(Data),
    String(Data),
    Boolean(Data),

    // defined by span rather than be contents
    Symbol,
    Label,

    // Operators
    Add, Sub,
    Mul, Div, Rem,
    Pow,

    Equal,

    // EoS
    End,
}

impl Display for Token {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        // pretty formatting for tokens
        // just use debug if you're not printing a message or something.
        let message = match self {
            Token::OpenBracket  => "an opening bracket",
            Token::CloseBracket => "a closing bracket",
            Token::OpenParen    => "an openening paren",
            Token::CloseParen   => "a closing paren",
            Token::Sep          => "a separator",
            Token::Syntax       => "a syntax definition",
            Token::Assign       => "an assignment",
            Token::Lambda       => "a lambda",
            Token::Compose      => "a composition",
            Token::Unit         => "the Unit, '()'",
            Token::Pair         => "a tuple",
            Token::Magic        => "a magic keyword",
            Token::Symbol       => "a symbol",
            Token::Label        => "a Label", // capitilized to mimic actual labels
            Token::Number(_)    => "a number",
            Token::String(_)    => "a string",
            Token::Add          => "an addition",
            Token::Sub          => "a subtraction",
            Token::Mul          => "a multiplication",
            Token::Div          => "a division",
            Token::Rem          => "a remainder",
            Token::Pow          => "a power of",
            Token::Equal        => "an equality test",
            Token::End          => "end of source",
            Token::Keyword(k) => { return write!(f, "the pseudokeyword '{}", k); },
            Token::Boolean(b) => { return write!(f, "the boolean {}",        b); },
        };
        write!(f, "{}", message)
    }
}