oak_j/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2
3/// J token type
4#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
5#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
6pub enum JTokenType {
7    /// Whitespace
8    Whitespace,
9    /// Newline
10    Newline,
11    /// Comment
12    Comment,
13
14    /// String literal
15    StringLiteral,
16    /// Number literal
17    NumberLiteral,
18    /// Identifier
19    Identifier,
20
21    // J primitives (Verbs, Adverbs, Conjunctions)
22    // Basic symbols
23    /// Equal (=)
24    Equal,
25    /// Dot (.)
26    Dot,
27    /// Colon (:)
28    Colon,
29
30    // Assignment
31    /// Global assignment (=:)
32    IsGlobal,
33    /// Local assignment (=.)
34    IsLocal,
35
36    // Common verbs
37    /// Plus (+)
38    Plus,
39    /// Minus (-)
40    Minus,
41    /// Star (*)
42    Star,
43    /// Percent (%)
44    Percent,
45    /// Dollar ($)
46    Dollar,
47    /// Comma (,)
48    Comma,
49    /// Hash (#)
50    Hash,
51    /// Slash (/)
52    Slash,
53    /// Backslash (\)
54    Backslash,
55    /// Pipe (|)
56    Pipe,
57    /// Ampersand (&)
58    Ampersand,
59    /// Caret (^)
60    Caret,
61    /// Tilde (~)
62    Tilde,
63    /// Less (<)
64    Less,
65    /// Greater (>)
66    Greater,
67
68    // Parentheses
69    /// Left parenthesis (()
70    LeftParen,
71    /// Right parenthesis ())
72    RightParen,
73    /// Left bracket ([)
74    LeftBracket,
75    /// Right bracket (])
76    RightBracket,
77    /// Left brace ({)
78    LeftBrace,
79    /// Right brace (})
80    RightBrace,
81
82    // Special
83    /// End of file
84    Eof,
85    /// Error unit
86    Error,
87}
88
89impl JTokenType {
90    /// Is keyword (J mostly uses primitives)
91    pub fn is_keyword(&self) -> bool {
92        false
93    }
94
95    /// Is punctuation
96    pub fn is_punctuation(&self) -> bool {
97        matches!(self, Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace)
98    }
99
100    /// Is ignored token (whitespace or comment)
101    pub fn is_ignored(&self) -> bool {
102        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
103    }
104}
105
106impl TokenType for JTokenType {
107    type Role = UniversalTokenRole;
108    const END_OF_STREAM: Self = Self::Eof;
109
110    fn role(&self) -> Self::Role {
111        match self {
112            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
113            Self::Comment => UniversalTokenRole::Comment,
114            Self::StringLiteral | Self::NumberLiteral => UniversalTokenRole::Literal,
115            Self::Identifier => UniversalTokenRole::Name,
116            _ if self.is_punctuation() => UniversalTokenRole::Punctuation,
117            Self::Eof | Self::Error => UniversalTokenRole::None,
118            _ => UniversalTokenRole::Operator,
119        }
120    }
121}
oak_j/lexer/token_type.rs

oak_j/lexer/
token_type.rs
