oak_j/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// J token type
6#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
7#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
8pub enum JTokenType {
9    /// Whitespace
10    Whitespace,
11    /// Newline
12    Newline,
13    /// Comment
14    Comment,
15
16    /// String literal
17    StringLiteral,
18    /// Number literal
19    NumberLiteral,
20    /// Identifier
21    Identifier,
22
23    // J primitives (Verbs, Adverbs, Conjunctions)
24    // Basic symbols
25    /// Equal (=)
26    Equal,
27    /// Dot (.)
28    Dot,
29    /// Colon (:)
30    Colon,
31
32    // Assignment
33    /// Global assignment (=:)
34    IsGlobal,
35    /// Local assignment (=.)
36    IsLocal,
37
38    // Common verbs
39    /// Plus (+)
40    Plus,
41    /// Minus (-)
42    Minus,
43    /// Star (*)
44    Star,
45    /// Percent (%)
46    Percent,
47    /// Dollar ($)
48    Dollar,
49    /// Comma (,)
50    Comma,
51    /// Hash (#)
52    Hash,
53    /// Slash (/)
54    Slash,
55    /// Backslash (\)
56    Backslash,
57    /// Pipe (|)
58    Pipe,
59    /// Ampersand (&)
60    Ampersand,
61    /// Caret (^)
62    Caret,
63    /// Tilde (~)
64    Tilde,
65    /// Less (<)
66    Less,
67    /// Greater (>)
68    Greater,
69
70    // Parentheses
71    /// Left parenthesis (()
72    LeftParen,
73    /// Right parenthesis ())
74    RightParen,
75    /// Left bracket ([)
76    LeftBracket,
77    /// Right bracket (])
78    RightBracket,
79    /// Left brace ({)
80    LeftBrace,
81    /// Right brace (})
82    RightBrace,
83
84    // Special
85    /// End of file
86    Eof,
87    /// Error unit
88    Error,
89}
90
91impl JTokenType {
92    /// Is keyword (J mostly uses primitives)
93    pub fn is_keyword(&self) -> bool {
94        false
95    }
96
97    /// Is punctuation
98    pub fn is_punctuation(&self) -> bool {
99        matches!(self, Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace)
100    }
101
102    /// Is ignored token (whitespace or comment)
103    pub fn is_ignored(&self) -> bool {
104        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
105    }
106}
107
108impl TokenType for JTokenType {
109    type Role = UniversalTokenRole;
110    const END_OF_STREAM: Self = Self::Eof;
111
112    fn role(&self) -> Self::Role {
113        match self {
114            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
115            Self::Comment => UniversalTokenRole::Comment,
116            Self::StringLiteral | Self::NumberLiteral => UniversalTokenRole::Literal,
117            Self::Identifier => UniversalTokenRole::Name,
118            _ if self.is_punctuation() => UniversalTokenRole::Punctuation,
119            Self::Eof | Self::Error => UniversalTokenRole::None,
120            _ => UniversalTokenRole::Operator,
121        }
122    }
123}
oak_j/lexer/token_type.rs

oak_j/lexer/
token_type.rs
