varyk_syntax/
token.rs

1//! Token kinds produced by the lexer.
2
3/// The kind of a lexical token. Keywords that are part of the Varyk
4/// surface (spec 4.1) each get their own variant; every other Rust keyword
5/// and reserved word, including the 2024-edition ones, lexes as
6/// [`TokenKind::ReservedKeyword`] carrying its text, so the parser can report
7/// it by name.
8#[derive(Debug, Clone, PartialEq, Eq)]
9pub enum TokenKind {
10    // Keywords (spec 4.1). `true` and `false` are bool literals, not
11    // keyword tokens, so they are not listed here.
12    Fn,
13    Pub,
14    Let,
15    Mut,
16    Struct,
17    Mod,
18    If,
19    Else,
20    While,
21    Break,
22    Continue,
23    Return,
24    Enum,
25    Impl,
26    Match,
27    For,
28    In,
29    /// `self`, spec 2.1, also a path prefix (spec 3.1, 3.3). `Self` is
30    /// unrelated and still lexes as [`TokenKind::ReservedKeyword`]: Varyk
31    /// has no way to write it.
32    SelfKw,
33    /// `crate`, a path prefix naming the crate root (spec 3.1, 3.3).
34    CrateKw,
35    /// `super`, a path prefix naming the parent module (spec 3.1, 3.3).
36    SuperKw,
37    /// `use`, spec 3.3.
38    UseKw,
39    /// `as`: a cast, `expr as T` (M4 spec 2.9), and the alias in
40    /// `use path as name;` (spec 3.3).
41    As,
42
43    /// Any other Rust keyword or reserved word, carrying its text.
44    ReservedKeyword(String),
45
46    Identifier(String),
47
48    /// Raw source text of an integer literal, digits only.
49    IntegerLiteral(String),
50    /// Raw source text of a float literal, digits `.` digits.
51    FloatLiteral(String),
52    /// Raw source text between the quotes of a string literal, unmodified.
53    StringLiteral(String),
54    BoolLiteral(bool),
55
56    /// A `'ident` lifetime, carrying the identifier without the quote.
57    Lifetime(String),
58
59    // Operators and punctuation (spec 4.1).
60    Plus,
61    Minus,
62    Star,
63    Slash,
64    Percent,
65    Eq,
66    EqEq,
67    NotEq,
68    Lt,
69    LtEq,
70    Gt,
71    GtEq,
72    Bang,
73    AmpAmp,
74    PipePipe,
75    /// A lone `|`, around a closure's parameters (M4 spec 2.2).
76    Pipe,
77    Amp,
78    ColonColon,
79    Colon,
80    Semi,
81    Comma,
82    Dot,
83    DotDot,
84    /// `..=`, an inclusive range (M4 spec 2.5, 2.11).
85    DotDotEq,
86    LParen,
87    RParen,
88    LBrace,
89    RBrace,
90    LBracket,
91    RBracket,
92    Arrow,
93    FatArrow,
94    Question,
95}
96
97/// A single lexical token: its kind plus the span it came from.
98#[derive(Debug, Clone, PartialEq, Eq)]
99pub struct Token {
100    pub kind: TokenKind,
101    pub span: crate::span::Span,
102}
varyk_syntax/token.rs

varyk_syntax/
token.rs
