oak_j/lexer/
token_type.rs

1use oak_core::{TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// J 词法单元类型
6#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
7#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
8pub enum JTokenType {
9    Whitespace,
10    Newline,
11    Comment,
12
13    StringLiteral,
14    NumberLiteral,
15    Identifier,
16
17    // J 原始操作符 (Verbs, Adverbs, Conjunctions)
18    // 基础符号
19    Equal, // =
20    Dot,   // .
21    Colon, // :
22
23    // 赋值
24    IsGlobal, // =:
25    IsLocal,  // =.
26
27    // 常用动词
28    Plus,      // +
29    Minus,     // -
30    Star,      // *
31    Percent,   // %
32    Dollar,    // $
33    Comma,     // ,
34    Hash,      // #
35    Slash,     // /
36    Backslash, // \
37    Pipe,      // |
38    Ampersand, // &
39    Caret,     // ^
40    Tilde,     // ~
41    Less,      // <
42    Greater,   // >
43
44    // 括号
45    LeftParen,    // (
46    RightParen,   // )
47    LeftBracket,  // [
48    RightBracket, // ]
49    LeftBrace,    // {
50    RightBrace,   // }
51
52    // 特殊
53    Eof,
54    Error,
55}
56
57impl JTokenType {
58    /// 是否为关键字 (J 中更多是原始操作符)
59    pub fn is_keyword(&self) -> bool {
60        false
61    }
62
63    /// 是否为标点符号
64    pub fn is_punctuation(&self) -> bool {
65        matches!(self, Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace)
66    }
67
68    /// 是否为忽略的单元 (空白或注释)
69    pub fn is_ignored(&self) -> bool {
70        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
71    }
72}
73
74impl TokenType for JTokenType {
75    type Role = UniversalTokenRole;
76    const END_OF_STREAM: Self = Self::Eof;
77
78    fn role(&self) -> Self::Role {
79        match self {
80            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
81            Self::Comment => UniversalTokenRole::Comment,
82            Self::StringLiteral | Self::NumberLiteral => UniversalTokenRole::Literal,
83            Self::Identifier => UniversalTokenRole::Name,
84            _ if self.is_punctuation() => UniversalTokenRole::Punctuation,
85            Self::Eof | Self::Error => UniversalTokenRole::None,
86            _ => UniversalTokenRole::Operator,
87        }
88    }
89}
oak_j/lexer/token_type.rs

oak_j/lexer/
token_type.rs
