oak_scheme/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5pub type SchemeToken = Token<SchemeTokenType>;
6
7impl TokenType for SchemeTokenType {
8    type Role = UniversalTokenRole;
9    const END_OF_STREAM: Self = Self::Eof;
10
11    fn is_ignored(&self) -> bool {
12        matches!(self, Self::Whitespace | Self::Newline | Self::Comment | Self::LineComment)
13    }
14
15    fn role(&self) -> Self::Role {
16        match self {
17            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
18            Self::Comment | Self::LineComment => UniversalTokenRole::Comment,
19            Self::Error => UniversalTokenRole::Error,
20            Self::Eof => UniversalTokenRole::Eof,
21            _ => UniversalTokenRole::None,
22        }
23    }
24}
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
27#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
28pub enum SchemeTokenType {
29    // 空白字符和换行
30    Whitespace,
31    Newline,
32    Comment,
33
34    // 注释
35    LineComment,
36
37    // 字面量
38    NumberLiteral,
39    StringLiteral,
40    CharacterLiteral,
41    BooleanLiteral,
42
43    // 标识符和符号
44    Identifier,
45    Symbol,
46
47    // 关键字
48    Keyword,
49    Define,
50    Lambda,
51    If,
52    Cond,
53    Case,
54    Let,
55    LetStar,
56    Letrec,
57    Begin,
58    Do,
59    Quote,
60    Quasiquote,
61    Unquote,
62    UnquoteSplicing,
63    And,
64    Or,
65    Not,
66    Set,
67
68    // 分隔符
69    LeftParen,
70    RightParen,
71    LeftBracket,
72    RightBracket,
73    LeftBrace,
74    RightBrace,
75    Dot,
76
77    // 特殊符号
78    Hash,
79    Quote_,
80    Quasiquote_,
81    Unquote_,
82    UnquoteSplicing_,
83
84    // 错误和结束
85    Error,
86    Eof,
87
88    // 根节点
89    SourceFile,
90}
oak_scheme/lexer/token_type.rs

oak_scheme/lexer/
token_type.rs
