varyk-syntax 0.1.0

Lexer, parser, and AST for Varyk, a programming language for backend services that compiles to Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! Token kinds produced by the lexer.

/// The kind of a lexical token. Keywords that are part of the Varyk
/// surface (spec 4.1) each get their own variant; every other Rust keyword
/// and reserved word, including the 2024-edition ones, lexes as
/// [`TokenKind::ReservedKeyword`] carrying its text, so the parser can report
/// it by name.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TokenKind {
    // Keywords (spec 4.1). `true` and `false` are bool literals, not
    // keyword tokens, so they are not listed here.
    Fn,
    Pub,
    Let,
    Mut,
    Struct,
    Mod,
    If,
    Else,
    While,
    Break,
    Continue,
    Return,
    Enum,
    Impl,
    Match,
    For,
    In,
    /// `self`, spec 2.1, also a path prefix (spec 3.1, 3.3). `Self` is
    /// unrelated and still lexes as [`TokenKind::ReservedKeyword`]: Varyk
    /// has no way to write it.
    SelfKw,
    /// `crate`, a path prefix naming the crate root (spec 3.1, 3.3).
    CrateKw,
    /// `super`, a path prefix naming the parent module (spec 3.1, 3.3).
    SuperKw,
    /// `use`, spec 3.3.
    UseKw,

    /// Any other Rust keyword or reserved word, carrying its text.
    ReservedKeyword(String),

    Identifier(String),

    /// Raw source text of an integer literal, digits only.
    IntegerLiteral(String),
    /// Raw source text of a float literal, digits `.` digits.
    FloatLiteral(String),
    /// Raw source text between the quotes of a string literal, unmodified.
    StringLiteral(String),
    BoolLiteral(bool),

    /// A `'ident` lifetime, carrying the identifier without the quote.
    Lifetime(String),

    // Operators and punctuation (spec 4.1).
    Plus,
    Minus,
    Star,
    Slash,
    Percent,
    Eq,
    EqEq,
    NotEq,
    Lt,
    LtEq,
    Gt,
    GtEq,
    Bang,
    AmpAmp,
    PipePipe,
    Amp,
    ColonColon,
    Colon,
    Semi,
    Comma,
    Dot,
    DotDot,
    LParen,
    RParen,
    LBrace,
    RBrace,
    LBracket,
    RBracket,
    Arrow,
    FatArrow,
    Question,
}

/// A single lexical token: its kind plus the span it came from.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
    pub kind: TokenKind,
    pub span: crate::span::Span,
}