yog_script_analyze/source/
token.rs

1//! What the lexer cuts a source into: tokens, each with where it is.
2
3use crate::words::Word;
4
5/// Where something is in its source: byte offsets, `start..end`.
6pub use yog_data::Span;
7
8/// A part of a string literal: text as written, or `{expr}` cut into tokens.
9#[derive(Debug, Clone, PartialEq)]
10pub enum StrPart {
11    Text(String),
12    Expr(Vec<Token>),
13}
14
15/// What a token is.
16#[derive(Debug, Clone, PartialEq)]
17pub enum Tok {
18    Ident(String),
19    /// A reserved word (`words/`).
20    Keyword(&'static Word),
21    Int(i64),
22    Float(f64),
23    /// A number with a unit (`20t`, `3b`, `100Energy`): the number as
24    /// written, the unit's name.
25    Unit(String, String),
26    /// A version (`1.20.1`), as written.
27    Version(String),
28    Str(Vec<StrPart>),
29    RawStr(String),
30    /// A translatable text by key: `t"charge.tooltip"`.
31    Text(String),
32    /// `@kind:namespace:path`: the kind, the id.
33    Content(String, String),
34    /// `#namespace:path`.
35    Tag(String),
36    /// `/// …`: a declaration's documentation line.
37    Doc(String),
38    Punct(Punct),
39    /// The end of a statement (a newline that isn't a continuation).
40    Newline,
41    Eof,
42}
43
44/// A token and where it is.
45#[derive(Debug, Clone, PartialEq)]
46pub struct Token {
47    pub tok: Tok,
48    pub span: Span,
49}
50
51macro_rules! puncts {
52    ($(#[$doc:meta])* $name:ident { $($variant:ident = $text:literal,)* }) => {
53        $(#[$doc])*
54        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55        pub enum $name { $($variant),* }
56
57        impl $name {
58            pub const ALL: &'static [$name] = &[$($name::$variant),*];
59
60            /// How it's written.
61            #[must_use]
62            pub const fn text(self) -> &'static str {
63                match self { $($name::$variant => $text),* }
64            }
65        }
66    };
67}
68
69puncts! {
70    /// Punctuation and operators, longest first where one starts another.
71    Punct {
72        RangeExcl = "..<", NullAssign = "??=",
73        Arrow = "->", FatArrow = "=>", SafeDot = "?.", Coalesce = "??",
74        EqEq = "==", NotEq = "!=", LtEq = "<=", GtEq = ">=", AndAnd = "&&", OrOr = "||",
75        PlusEq = "+=", MinusEq = "-=", StarEq = "*=", SlashEq = "/=", PercentEq = "%=",
76        Range = "..",
77        LBrace = "{", RBrace = "}", LParen = "(", RParen = ")", LBracket = "[", RBracket = "]",
78        Comma = ",", Dot = ".", Colon = ":", Semi = ";", Question = "?",
79        Eq = "=", Lt = "<", Gt = ">", Plus = "+", Minus = "-", Star = "*", Slash = "/",
80        Percent = "%", Bang = "!", Pipe = "|", Hash = "#",
81    }
82}
83
84impl Punct {
85    /// After it a newline doesn't end the statement: the line goes on.
86    #[must_use]
87    pub const fn continues(self) -> bool {
88        !matches!(
89            self,
90            Punct::RBrace | Punct::RParen | Punct::RBracket | Punct::Semi | Punct::Question
91        )
92    }
93}
yog_script_analyze/source/token.rs

yog_script_analyze/source/
token.rs
