varyk_syntax/
token.rs

1//! Token kinds produced by the lexer.
2
3/// The kind of a lexical token. Keywords that are part of the Varyk
4/// surface (spec 4.1) each get their own variant; every other Rust keyword
5/// and reserved word, including the 2024-edition ones, lexes as
6/// [`TokenKind::ReservedKeyword`] carrying its text, so the parser can report
7/// it by name.
8#[derive(Debug, Clone, PartialEq, Eq)]
9pub enum TokenKind {
10    // Keywords (spec 4.1). `true` and `false` are bool literals, not
11    // keyword tokens, so they are not listed here.
12    Fn,
13    Pub,
14    Let,
15    Mut,
16    Struct,
17    Mod,
18    If,
19    Else,
20    While,
21    Break,
22    Continue,
23    Return,
24    Enum,
25    Impl,
26    Match,
27    For,
28    In,
29    /// `self`, spec 2.1. `Self` is unrelated and still lexes as
30    /// [`TokenKind::ReservedKeyword`]: milestone 2 has no way to write it.
31    SelfKw,
32
33    /// Any other Rust keyword or reserved word, carrying its text.
34    ReservedKeyword(String),
35
36    Identifier(String),
37
38    /// Raw source text of an integer literal, digits only.
39    IntegerLiteral(String),
40    /// Raw source text of a float literal, digits `.` digits.
41    FloatLiteral(String),
42    /// Raw source text between the quotes of a string literal, unmodified.
43    StringLiteral(String),
44    BoolLiteral(bool),
45
46    /// A `'ident` lifetime, carrying the identifier without the quote.
47    Lifetime(String),
48
49    // Operators and punctuation (spec 4.1).
50    Plus,
51    Minus,
52    Star,
53    Slash,
54    Percent,
55    Eq,
56    EqEq,
57    NotEq,
58    Lt,
59    LtEq,
60    Gt,
61    GtEq,
62    Bang,
63    AmpAmp,
64    PipePipe,
65    Amp,
66    ColonColon,
67    Colon,
68    Semi,
69    Comma,
70    Dot,
71    DotDot,
72    LParen,
73    RParen,
74    LBrace,
75    RBrace,
76    LBracket,
77    RBracket,
78    Arrow,
79    FatArrow,
80    Question,
81}
82
83/// A single lexical token: its kind plus the span it came from.
84#[derive(Debug, Clone, PartialEq, Eq)]
85pub struct Token {
86    pub kind: TokenKind,
87    pub span: crate::span::Span,
88}
varyk_syntax/token.rs

varyk_syntax/
token.rs
