oak_clojure/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2use serde::{Deserialize, Serialize};
3
4/// Represents all possible token kinds in the Clojure programming language.
5#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
6pub enum ClojureTokenType {
7    // Trivia
8    /// Whitespace characters (spaces, tabs)
9    Whitespace,
10    /// Newline characters
11    Newline,
12    /// Comments (both single-line and multi-line)
13    Comment,
14
15    // Literals
16    /// String literals (e.g., "hello")
17    StringLiteral,
18    /// Character literals (e.g., \a)
19    CharacterLiteral,
20    /// Number literals (integer and floating-point)
21    NumberLiteral,
22    /// Boolean literals (true, false)
23    BooleanLiteral,
24    /// Nil literal
25    NilLiteral,
26    /// Keyword literals (e.g., :keyword)
27    KeywordLiteral,
28
29    // Identifiers and symbols
30    /// Symbol identifiers (e.g., variable names, function names)
31    Symbol,
32    /// Keyword identifiers (e.g., :keyword)
33    Keyword,
34
35    // Collections
36    /// List start delimiter: `(`
37    ListStart,
38    /// List end delimiter: `)`
39    ListEnd,
40    /// Vector start delimiter: `[`
41    VectorStart,
42    /// Vector end delimiter: `]`
43    VectorEnd,
44    /// Map start delimiter: `{`
45    MapStart,
46    /// Map end delimiter: `}`
47    MapEnd,
48    /// Set start delimiter: `#{`
49    SetStart,
50
51    // Special forms
52    /// Quote form (e.g., 'expr)
53    Quote,
54    /// Unquote form (e.g., ~expr)
55    Unquote,
56    /// Unquote-splicing form (e.g., ~@expr)
57    UnquoteSplice,
58    /// Deref form (e.g., @expr)
59    Deref,
60    /// Metadata form (e.g., ^metadata expr)
61    Meta,
62    /// Dispatch macro (e.g., #)
63    Dispatch,
64
65    // Reader macros
66    /// Reader macro form (e.g., #tag expr)
67    ReaderMacro,
68
69    // Regex
70    /// Regular expression literals (e.g., #"pattern")
71    RegexLiteral,
72
73    // Anonymous function
74    /// Anonymous function start delimiter: #(
75    AnonFnStart,
76    /// Anonymous function argument (e.g., %1, %2, etc.)
77    AnonFnArg,
78
79    // Error handling
80    /// Error token
81    Error,
82    /// End of file marker
83    Eof,
84}
85
86impl TokenType for ClojureTokenType {
87    type Role = UniversalTokenRole;
88    const END_OF_STREAM: Self = Self::Eof;
89
90    fn is_ignored(&self) -> bool {
91        self.is_whitespace() || self.is_comment()
92    }
93
94    fn is_comment(&self) -> bool {
95        matches!(self, Self::Comment)
96    }
97
98    fn is_whitespace(&self) -> bool {
99        matches!(self, Self::Whitespace | Self::Newline)
100    }
101
102    fn role(&self) -> Self::Role {
103        match self {
104            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
105            Self::Comment => UniversalTokenRole::Comment,
106            Self::StringLiteral | Self::CharacterLiteral | Self::NumberLiteral | Self::BooleanLiteral | Self::NilLiteral | Self::KeywordLiteral | Self::RegexLiteral => UniversalTokenRole::Literal,
107            Self::Symbol | Self::Keyword | Self::AnonFnArg => UniversalTokenRole::Name,
108            Self::ListStart
109            | Self::ListEnd
110            | Self::VectorStart
111            | Self::VectorEnd
112            | Self::MapStart
113            | Self::MapEnd
114            | Self::SetStart
115            | Self::Quote
116            | Self::Unquote
117            | Self::UnquoteSplice
118            | Self::Deref
119            | Self::Meta
120            | Self::Dispatch
121            | Self::ReaderMacro
122            | Self::AnonFnStart => UniversalTokenRole::Punctuation,
123            Self::Error => UniversalTokenRole::Error,
124            Self::Eof => UniversalTokenRole::None,
125        }
126    }
127}
oak_clojure/lexer/token_type.rs

oak_clojure/lexer/
token_type.rs
