hiko_syntax/
token.rs

1use crate::span::Span;
2
3#[derive(Debug, Clone, PartialEq)]
4pub struct Token {
5    pub kind: TokenKind,
6    pub span: Span,
7}
8
9#[derive(Debug, Clone, PartialEq)]
10pub enum TokenKind {
11    // Literals
12    IntLit(i64),
13    FloatLit(f64),
14    StringLit(String),
15    CharLit(char),
16
17    // Identifiers
18    Ident(String),      // lowercase-initial
19    UpperIdent(String), // uppercase-initial (constructors)
20    TyVar(String),      // 'a, 'b, etc.
21
22    // Keywords
23    Val,
24    Fun,
25    Fn,
26    Let,
27    In,
28    End,
29    If,
30    Then,
31    Else,
32    Case,
33    Of,
34    Datatype,
35    Type,
36    Local,
37    And,
38    Rec,
39    Use,
40    True,
41    False,
42    Not,
43    Andalso,
44    Orelse,
45    Mod,
46    As,
47    Effect,
48    Handle,
49    With,
50    Perform,
51    Resume,
52    Return,
53
54    // Operators (Int arithmetic)
55    Plus,  // +
56    Minus, // -
57    Star,  // *
58    Slash, // /
59
60    // Operators (Float arithmetic)
61    PlusDot,  // +.
62    MinusDot, // -.
63    StarDot,  // *.
64    SlashDot, // /.
65
66    // Operators (String)
67    Caret, // ^
68
69    // Operators (Int comparison)
70    Lt, // <
71    Gt, // >
72    Le, // <=
73    Ge, // >=
74
75    // Operators (Float comparison)
76    LtDot, // <.
77    GtDot, // >.
78    LeDot, // <=.
79    GeDot, // >=.
80
81    // Operators (equality, scalar only)
82    Eq, // =
83    Ne, // <>
84
85    // Other operators
86    Tilde,      // ~ (unary negation)
87    ColonColon, // ::
88    Arrow,      // =>
89    ThinArrow,  // ->
90    Bar,        // |
91
92    // Delimiters
93    LParen,     // (
94    RParen,     // )
95    LBracket,   // [
96    RBracket,   // ]
97    Comma,      // ,
98    Colon,      // :
99    Semicolon,  // ;
100    Underscore, // _
101    Hash,       // #
102
103    // Special
104    Eof,
105}
106
107impl TokenKind {
108    pub fn keyword_from_str(s: &str) -> Option<TokenKind> {
109        match s {
110            "val" => Some(TokenKind::Val),
111            "fun" => Some(TokenKind::Fun),
112            "fn" => Some(TokenKind::Fn),
113            "let" => Some(TokenKind::Let),
114            "in" => Some(TokenKind::In),
115            "end" => Some(TokenKind::End),
116            "if" => Some(TokenKind::If),
117            "then" => Some(TokenKind::Then),
118            "else" => Some(TokenKind::Else),
119            "case" => Some(TokenKind::Case),
120            "of" => Some(TokenKind::Of),
121            "datatype" => Some(TokenKind::Datatype),
122            "type" => Some(TokenKind::Type),
123            "local" => Some(TokenKind::Local),
124            "and" => Some(TokenKind::And),
125            "rec" => Some(TokenKind::Rec),
126            "use" => Some(TokenKind::Use),
127            "true" => Some(TokenKind::True),
128            "false" => Some(TokenKind::False),
129            "not" => Some(TokenKind::Not),
130            "andalso" => Some(TokenKind::Andalso),
131            "orelse" => Some(TokenKind::Orelse),
132            "mod" => Some(TokenKind::Mod),
133            "as" => Some(TokenKind::As),
134            "effect" => Some(TokenKind::Effect),
135            "handle" => Some(TokenKind::Handle),
136            "with" => Some(TokenKind::With),
137            "perform" => Some(TokenKind::Perform),
138            "resume" => Some(TokenKind::Resume),
139            "return" => Some(TokenKind::Return),
140            _ => None,
141        }
142    }
143}
hiko_syntax/token.rs

hiko_syntax/
token.rs
