oak_toml/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2
3/// Represents the different types of tokens in the TOML language.
4#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
5#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
6pub enum TomlTokenKind {
7    /// Spaces, tabs, and other non-newline whitespace.
8    Whitespace,
9    /// Newline characters (\n or \r\n).
10    Newline,
11    /// TOML comments starting with #.
12    Comment,
13
14    // Literals
15    /// A generic string literal.
16    String,
17    /// A basic string with escape sequences (surrounded by ").
18    BasicString,
19    /// A literal string without escape sequences (surrounded by ').
20    LiteralString,
21    /// An integer literal.
22    Integer,
23    /// A floating-point literal.
24    Float,
25    /// A boolean literal (true or false).
26    Boolean,
27    /// An RFC 3339 offset date-time.
28    OffsetDateTime,
29    /// A local date-time without offset.
30    LocalDateTime,
31    /// A local date without time.
32    LocalDate,
33    /// A local time without date.
34    LocalTime,
35
36    // Keywords/Identifiers
37    /// A generic identifier.
38    Identifier,
39    /// A bare key used in key-value pairs.
40    BareKey,
41
42    // Symbols
43    /// The equals sign (=) used for assignment.
44    Equal, // =
45    /// The dot (.) used for nested keys.
46    Dot, // .
47    /// The comma (,) used as a separator in arrays or inline tables.
48    Comma, // ,
49    /// The left brace ({) for inline tables.
50    LeftBrace, // {
51    /// The right brace (}) for inline tables.
52    RightBrace, // }
53    /// The left bracket ([) for arrays or tables.
54    LeftBracket, // [
55    /// The right bracket (]) for arrays or tables.
56    RightBracket, // ]
57    /// Double left brackets ([[) for array of tables.
58    DoubleLeftBracket,
59    /// Double right brackets (]]) for array of tables.
60    DoubleRightBracket,
61
62    // Internal
63    /// Represents a lexing error.
64    Error,
65    /// Represents the end of the source stream.
66    Eof,
67    /// Represents the root of the TOML document.
68    Root,
69}
70
71/// Alias for `TomlTokenKind` to conform to Oak's naming conventions.
72pub type TomlTokenType = TomlTokenKind;
73
74impl TokenType for TomlTokenKind {
75    type Role = UniversalTokenRole;
76    const END_OF_STREAM: Self = Self::Eof;
77
78    fn role(&self) -> Self::Role {
79        match self {
80            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
81            Self::Comment => UniversalTokenRole::Comment,
82            Self::BasicString | Self::LiteralString | Self::String | Self::Integer | Self::Float | Self::Boolean | Self::OffsetDateTime | Self::LocalDateTime | Self::LocalDate | Self::LocalTime => UniversalTokenRole::Literal,
83            Self::BareKey | Self::Identifier => UniversalTokenRole::Name,
84            Self::Equal | Self::Dot | Self::Comma | Self::LeftBrace | Self::RightBrace | Self::LeftBracket | Self::RightBracket | Self::DoubleLeftBracket | Self::DoubleRightBracket => UniversalTokenRole::Punctuation,
85            Self::Root => UniversalTokenRole::None,
86            Self::Error => UniversalTokenRole::Error,
87            Self::Eof => UniversalTokenRole::None,
88        }
89    }
90}
oak_toml/lexer/token_type.rs

oak_toml/lexer/
token_type.rs
