yog_script_analyze/source/
token.rs

1//! What the lexer cuts a source into: tokens, each with where it is.
2
3use crate::words::Word;
4
5/// Where something is in its source: byte offsets, `start..end`.
6pub use yog_data::Span;
7
8/// What a character literal is between and a lifetime starts with
9/// (`'R'`, `'a`).
10pub const QUOTE: char = '\'';
11
12/// A part of a string literal: text as written, or `{expr}` cut into tokens.
13#[derive(Debug, Clone, PartialEq)]
14pub enum StrPart {
15    Text(String),
16    /// `{expr,alignment:format}`: the expression's tokens, then what pads
17    /// and formats its value, as written (C#'s composite format).
18    Expr {
19        tokens: Vec<Token>,
20        alignment: Option<String>,
21        format: Option<String>,
22    },
23}
24
25/// What a token is.
26#[derive(Debug, Clone, PartialEq)]
27pub enum Tok {
28    Ident(String),
29    /// A reserved word (`words/`).
30    Keyword(&'static Word),
31    Int(i64),
32    Float(f64),
33    /// A number with a unit (`20t`, `3b`, `100Energy`): the number as
34    /// written, the unit's name.
35    Unit(String, String),
36    /// A version (`1.20.1`), as written.
37    Version(String),
38    Str(Vec<StrPart>),
39    RawStr(String),
40    Char(char),
41    /// `'a`: a lifetime, by its name.
42    Lifetime(String),
43    /// `/// …`: a declaration's documentation line.
44    Doc(String),
45    Punct(Punct),
46    Eof,
47}
48
49/// A token and where it is.
50#[derive(Debug, Clone, PartialEq)]
51pub struct Token {
52    pub tok: Tok,
53    pub span: Span,
54}
55
56macro_rules! puncts {
57    ($(#[$doc:meta])* $name:ident { $($variant:ident = $text:literal,)* }) => {
58        $(#[$doc])*
59        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
60        pub enum $name { $($variant),* }
61
62        impl $name {
63            pub const ALL: &'static [$name] = &[$($name::$variant),*];
64
65            /// How it's written.
66            #[must_use]
67            pub const fn text(self) -> &'static str {
68                match self { $($name::$variant => $text),* }
69            }
70        }
71    };
72}
73
74puncts! {
75    /// Punctuation and operators, longest first where one starts another.
76    Punct {
77        RangeExcl = "..<", NullAssign = "??=", ShlEq = "<<=",
78        FatArrow = "=>", SafeDot = "?.", Coalesce = "??",
79        EqEq = "==", NotEq = "!=", Shl = "<<", LtEq = "<=", GtEq = ">=", AndAnd = "&&", OrOr = "||",
80        PlusEq = "+=", MinusEq = "-=", StarEq = "*=", SlashEq = "/=", PercentEq = "%=",
81        AmpEq = "&=", PipeEq = "|=", CaretEq = "^=", PlusPlus = "++", MinusMinus = "--",
82        Range = "..",
83        LBrace = "{", RBrace = "}", LParen = "(", RParen = ")", LBracket = "[", RBracket = "]",
84        Comma = ",", Dot = ".", Colon = ":", Semi = ";", Question = "?",
85        Eq = "=", Lt = "<", Gt = ">", Plus = "+", Minus = "-", Star = "*", Slash = "/",
86        Percent = "%", Bang = "!", Amp = "&", Hash = "#", Dollar = "$",
87        Pipe = "|", Caret = "^", Tilde = "~",
88    }
89}
yog_script_analyze/source/token.rs

yog_script_analyze/source/
token.rs
