varyk_syntax/
token.rs

1//! Token kinds produced by the lexer.
2
3/// The kind of a lexical token. Keywords that are part of the Varyk
4/// surface (spec 4.1) each get their own variant; every other Rust keyword
5/// and reserved word, including the 2024-edition ones, lexes as
6/// [`TokenKind::ReservedKeyword`] carrying its text, so the parser can report
7/// it by name.
8#[derive(Debug, Clone, PartialEq, Eq)]
9pub enum TokenKind {
10    // Keywords (spec 4.1). `true` and `false` are bool literals, not
11    // keyword tokens, so they are not listed here.
12    Fn,
13    Pub,
14    Let,
15    Mut,
16    Struct,
17    Mod,
18    If,
19    Else,
20    While,
21    Break,
22    Continue,
23    Return,
24    Enum,
25    Impl,
26    Match,
27    For,
28    In,
29    /// `self`, spec 2.1, also a path prefix (spec 3.1, 3.3). `Self` is
30    /// unrelated and still lexes as [`TokenKind::ReservedKeyword`]: Varyk
31    /// has no way to write it.
32    SelfKw,
33    /// `crate`, a path prefix naming the crate root (spec 3.1, 3.3).
34    CrateKw,
35    /// `super`, a path prefix naming the parent module (spec 3.1, 3.3).
36    SuperKw,
37    /// `use`, spec 3.3.
38    UseKw,
39    /// `as`: a cast, `expr as T` (M4 spec 2.9), and the alias in
40    /// `use path as name;` (spec 3.3).
41    As,
42    /// `async`, before `fn` (milestone 5b1 spec 2.2).
43    Async,
44    /// `await`, only after a `.` (milestone 5b1 spec 2.3).
45    Await,
46
47    /// Any other Rust keyword or reserved word, carrying its text.
48    ReservedKeyword(String),
49
50    Identifier(String),
51
52    /// Raw source text of an integer literal, digits only.
53    IntegerLiteral(String),
54    /// Raw source text of a float literal, digits `.` digits.
55    FloatLiteral(String),
56    /// Raw source text between the quotes of a string literal, unmodified.
57    StringLiteral(String),
58    BoolLiteral(bool),
59
60    /// A `'ident` lifetime, carrying the identifier without the quote.
61    Lifetime(String),
62
63    // Operators and punctuation (spec 4.1).
64    Plus,
65    Minus,
66    Star,
67    Slash,
68    Percent,
69    Eq,
70    EqEq,
71    NotEq,
72    Lt,
73    LtEq,
74    Gt,
75    GtEq,
76    Bang,
77    AmpAmp,
78    PipePipe,
79    /// A lone `|`, around a closure's parameters (M4 spec 2.2).
80    Pipe,
81    Amp,
82    ColonColon,
83    Colon,
84    Semi,
85    Comma,
86    Dot,
87    DotDot,
88    /// `..=`, an inclusive range (M4 spec 2.5, 2.11).
89    DotDotEq,
90    LParen,
91    RParen,
92    LBrace,
93    RBrace,
94    LBracket,
95    RBracket,
96    Arrow,
97    FatArrow,
98    Question,
99    /// `#`, which starts an attribute, `#[name]` or `#[name(literal)]`
100    /// (M5a spec 2.1, 2.2).
101    Hash,
102}
103
104/// A single lexical token: its kind plus the span it came from.
105#[derive(Debug, Clone, PartialEq, Eq)]
106pub struct Token {
107    pub kind: TokenKind,
108    pub span: crate::span::Span,
109}
varyk_syntax/token.rs

varyk_syntax/
token.rs
