yog_script_analyze/source/
token.rs

1//! What the lexer cuts a source into: tokens, each with where it is.
2
3use crate::words::Word;
4
5/// Where something is in its source: byte offsets, `start..end`.
6pub use yog_data::Span;
7
8/// What a character literal is between and a lifetime starts with
9/// (`'R'`, `'a`).
10pub const QUOTE: char = '\'';
11
12/// A part of a string literal: text as written, or `{expr}` cut into tokens.
13#[derive(Debug, Clone, PartialEq)]
14pub enum StrPart {
15    Text(String),
16    Expr(Vec<Token>),
17}
18
19/// What a token is.
20#[derive(Debug, Clone, PartialEq)]
21pub enum Tok {
22    Ident(String),
23    /// A reserved word (`words/`).
24    Keyword(&'static Word),
25    Int(i64),
26    Float(f64),
27    /// A number with a unit (`20t`, `3b`, `100Energy`): the number as
28    /// written, the unit's name.
29    Unit(String, String),
30    /// A version (`1.20.1`), as written.
31    Version(String),
32    Str(Vec<StrPart>),
33    RawStr(String),
34    Char(char),
35    /// `'a`: a lifetime, by its name.
36    Lifetime(String),
37    /// A translatable text by key: `t"charge.tooltip"`.
38    Text(String),
39    /// `@kind:namespace:path`: the kind, the id.
40    Content(String, String),
41    /// `#namespace:path`.
42    Tag(String),
43    /// `/// …`: a declaration's documentation line.
44    Doc(String),
45    Punct(Punct),
46    Eof,
47}
48
49/// A token and where it is.
50#[derive(Debug, Clone, PartialEq)]
51pub struct Token {
52    pub tok: Tok,
53    pub span: Span,
54}
55
56macro_rules! puncts {
57    ($(#[$doc:meta])* $name:ident { $($variant:ident = $text:literal,)* }) => {
58        $(#[$doc])*
59        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
60        pub enum $name { $($variant),* }
61
62        impl $name {
63            pub const ALL: &'static [$name] = &[$($name::$variant),*];
64
65            /// How it's written.
66            #[must_use]
67            pub const fn text(self) -> &'static str {
68                match self { $($name::$variant => $text),* }
69            }
70        }
71    };
72}
73
74puncts! {
75    /// Punctuation and operators, longest first where one starts another.
76    Punct {
77        RangeExcl = "..<", NullAssign = "??=",
78        FatArrow = "=>", SafeDot = "?.", Coalesce = "??",
79        EqEq = "==", NotEq = "!=", LtEq = "<=", GtEq = ">=", AndAnd = "&&", OrOr = "||",
80        PlusEq = "+=", MinusEq = "-=", StarEq = "*=", SlashEq = "/=", PercentEq = "%=",
81        Range = "..",
82        LBrace = "{", RBrace = "}", LParen = "(", RParen = ")", LBracket = "[", RBracket = "]",
83        Comma = ",", Dot = ".", Colon = ":", Semi = ";", Question = "?",
84        Eq = "=", Lt = "<", Gt = ">", Plus = "+", Minus = "-", Star = "*", Slash = "/",
85        Percent = "%", Bang = "!", Amp = "&", Hash = "#", Dollar = "$",
86    }
87}
yog_script_analyze/source/token.rs

yog_script_analyze/source/
token.rs
