1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

use crate::syntax::characters_to_string;
use crate::*;
use std::fmt;

#[derive(Debug, Clone, PartialEq)]
pub enum TokenKind {
    EOF,
    Unknown(u16),
    Whitespace(String),
    LineComment(String),

    AsKeyword,
    InKeyword,
    IsKeyword,
    OutKeyword,
    InoutKeyword,
    ClassKeyword,
    PrivateKeyword,
    PublicKeyword,
    NamespaceKeyword,
    SelfKeyword,
    ImportKeyword,
    ExportKeyword,
    PartialKeyword,
    LetKeyword,
    NativeKeyword,
    PanicKeyword,
    InitKeyword,
    VarKeyword,

    Dash,
    Plus,
    Colon,
    SemiColon,
    Comma,
    Period,
    Slash,
    EqualSign,
    Asterisk,

    Arrow,
    FatArrow,

    OpenAngle,
    CloseAngle,
    OpenCurly,
    CloseCurly,
    OpenBracket,
    CloseBracket,
    OpenParen,
    CloseParen,

    SimpleInteger(String),
    SimpleFloat(String),
    SimpleString(String),
    SimpleCharacter(String),
    SimpleSymbol(String),
    SymbolLiteral(String),

    Underscore,

    DocLineMarker,
    DocNewLine(String),
    DocText(String),
}

#[derive(Clone)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
    pub before: Vec<Token>,
    pub after: Vec<Token>,
}

impl Token {
    pub fn lexeme(&self) -> String {
        use TokenKind::*;

        match &self.kind {
            EOF => "\0".into(),
            Unknown(c) => characters_to_string([*c].iter().cloned()),

            AsKeyword => "as".into(),
            InKeyword => "in".into(),
            IsKeyword => "is".into(),
            OutKeyword => "out".into(),
            InoutKeyword => "inout".into(),
            ClassKeyword => "class".into(),
            PrivateKeyword => "private".into(),
            PublicKeyword => "public".into(),
            NamespaceKeyword => "namespace".into(),
            SelfKeyword => "self".into(),
            ImportKeyword => "import".into(),
            ExportKeyword => "export".into(),
            PartialKeyword => "partial".into(),
            LetKeyword => "let".into(),
            NativeKeyword => "native".into(),
            PanicKeyword => "panic".into(),
            InitKeyword => "init".into(),
            VarKeyword => "var".into(),

            Dash => "-".into(),
            Plus => "+".into(),
            Colon => ":".into(),
            SemiColon => ";".into(),
            Comma => ",".into(),
            Period => ".".into(),
            Slash => "/".into(),
            EqualSign => "=".into(),
            Asterisk => "*".into(),

            Arrow => "->".into(),
            FatArrow => "=>".into(),

            OpenAngle => "<".into(),
            CloseAngle => ">".into(),
            OpenCurly => "{".into(),
            CloseCurly => "}".into(),
            OpenBracket => "[".into(),
            CloseBracket => "]".into(),
            OpenParen => "(".into(),
            CloseParen => ")".into(),

            Underscore => "_".into(),

            LineComment(s) => format!("//{}", s),

            Whitespace(s) | SimpleString(s) | SimpleCharacter(s) | SimpleFloat(s)
            | SimpleInteger(s) | SimpleSymbol(s) | SymbolLiteral(s) | DocNewLine(s)
            | DocText(s) => s.clone(),

            DocLineMarker => "///".into(),
        }
    }
}

impl fmt::Debug for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.kind.fmt(f)
    }
}