yog_script_analyze/source/
token.rs

1//! What the lexer cuts a source into: tokens, each with where it is.
2
3use crate::words::Word;
4
5/// Where something is in its source: byte offsets, `start..end`.
6pub use yog_data::Span;
7
8/// A part of a string literal: text as written, or `{expr}` cut into tokens.
9#[derive(Debug, Clone, PartialEq)]
10pub enum StrPart {
11    Text(String),
12    Expr(Vec<Token>),
13}
14
15/// What a token is.
16#[derive(Debug, Clone, PartialEq)]
17pub enum Tok {
18    Ident(String),
19    /// A reserved word (`words/`).
20    Keyword(&'static Word),
21    Int(i64),
22    Float(f64),
23    /// A number with a unit (`20t`, `3b`, `100Energy`): the number as
24    /// written, the unit's name.
25    Unit(String, String),
26    /// A version (`1.20.1`), as written.
27    Version(String),
28    Str(Vec<StrPart>),
29    RawStr(String),
30    Char(char),
31    /// A translatable text by key: `t"charge.tooltip"`.
32    Text(String),
33    /// `@kind:namespace:path`: the kind, the id.
34    Content(String, String),
35    /// `#namespace:path`.
36    Tag(String),
37    /// `/// …`: a declaration's documentation line.
38    Doc(String),
39    Punct(Punct),
40    Eof,
41}
42
43/// A token and where it is.
44#[derive(Debug, Clone, PartialEq)]
45pub struct Token {
46    pub tok: Tok,
47    pub span: Span,
48}
49
50macro_rules! puncts {
51    ($(#[$doc:meta])* $name:ident { $($variant:ident = $text:literal,)* }) => {
52        $(#[$doc])*
53        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
54        pub enum $name { $($variant),* }
55
56        impl $name {
57            pub const ALL: &'static [$name] = &[$($name::$variant),*];
58
59            /// How it's written.
60            #[must_use]
61            pub const fn text(self) -> &'static str {
62                match self { $($name::$variant => $text),* }
63            }
64        }
65    };
66}
67
68puncts! {
69    /// Punctuation and operators, longest first where one starts another.
70    Punct {
71        RangeExcl = "..<", NullAssign = "??=",
72        FatArrow = "=>", SafeDot = "?.", Coalesce = "??",
73        EqEq = "==", NotEq = "!=", LtEq = "<=", GtEq = ">=", AndAnd = "&&", OrOr = "||",
74        PlusEq = "+=", MinusEq = "-=", StarEq = "*=", SlashEq = "/=", PercentEq = "%=",
75        Range = "..",
76        LBrace = "{", RBrace = "}", LParen = "(", RParen = ")", LBracket = "[", RBracket = "]",
77        Comma = ",", Dot = ".", Colon = ":", Semi = ";", Question = "?",
78        Eq = "=", Lt = "<", Gt = ">", Plus = "+", Minus = "-", Star = "*", Slash = "/",
79        Percent = "%", Bang = "!", Amp = "&", Hash = "#",
80    }
81}
yog_script_analyze/source/token.rs

yog_script_analyze/source/
token.rs
