oak_notedown/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// Token type for Notedown
4pub type NoteToken = Token<NoteTokenType>;
5
6impl TokenType for NoteTokenType {
7    type Role = UniversalTokenRole;
8    const END_OF_STREAM: Self = Self::Eof;
9
10    fn is_ignored(&self) -> bool {
11        matches!(self, Self::Whitespace)
12    }
13
14    fn role(&self) -> Self::Role {
15        match self {
16            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
17            Self::Text => UniversalTokenRole::None,
18            Self::Eof => UniversalTokenRole::Eof,
19            Self::Error => UniversalTokenRole::Error,
20            _ => UniversalTokenRole::None,
21        }
22    }
23}
24
25/// Enum representing all possible token types in Notedown
26#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
27#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
28#[repr(u8)]
29pub enum NoteTokenType {
30    /// Plain text
31    Text,
32    /// Whitespace characters
33    Whitespace,
34    /// Newline characters
35    Newline,
36
37    /// Level 1 heading marker (#)
38    Heading1,
39    /// Level 2 heading marker (##)
40    Heading2,
41    /// Level 3 heading marker (###)
42    Heading3,
43    /// Level 4 heading marker (####)
44    Heading4,
45    /// Level 5 heading marker (#####)
46    Heading5,
47    /// Level 6 heading marker (######)
48    Heading6,
49    /// Text content within a heading
50    HeadingText,
51
52    /// Emphasized text marker (* or _)
53    Emphasis,
54    /// Strong text marker (** or __)
55    Strong,
56    /// Strikethrough text marker (~~)
57    Strikethrough,
58
59    /// Inline code marker (`)
60    InlineCode,
61    /// Entire code block
62    CodeBlock,
63    /// Code block fence (``` or ~~~)
64    CodeFence,
65    /// Language identifier in a code block
66    CodeLanguage,
67
68    /// Link marker ([)
69    Link,
70    /// Text within a link
71    LinkText,
72    /// URL of a link
73    LinkUrl,
74    /// Title attribute of a link
75    LinkTitle,
76    /// Image marker (![)
77    Image,
78    /// Alt text of an image
79    ImageAlt,
80    /// URL of an image
81    ImageUrl,
82    /// Title attribute of an image
83    ImageTitle,
84
85    /// Unordered list container
86    UnorderedList,
87    /// Ordered list container
88    OrderedList,
89    /// Single list item
90    ListItem,
91    /// List marker (-, *, +, 1., etc.)
92    ListMarker,
93    /// Task list container
94    TaskList,
95    /// Task list marker ([ ] or [x])
96    TaskMarker,
97
98    /// Blockquote container
99    Blockquote,
100    /// Blockquote marker (>)
101    BlockquoteMarker,
102
103    /// Horizontal rule (---, ***, ___)
104    HorizontalRule,
105
106    /// Table container
107    Table,
108    /// Table row
109    TableRow,
110    /// Table cell
111    TableCell,
112    /// Table header cell
113    TableHeader,
114    /// Table separator (|)
115    TableSeparator,
116    /// Table alignment marker (:---, :---:, ---:)
117    TableAlignment,
118
119    /// HTML tag
120    HtmlTag,
121    /// HTML comment
122    HtmlComment,
123
124    /// Escape character (\)
125    Escape,
126
127    /// Left bracket ([)
128    LeftBracket,
129    /// Right bracket (])
130    RightBracket,
131    /// Left parenthesis (()
132    LeftParen,
133    /// Right parenthesis ())
134    RightParen,
135    /// Left angle bracket (<)
136    LeftAngle,
137    /// Right angle bracket (>)
138    RightAngle,
139    /// Asterisk (*)
140    Asterisk,
141    /// Underscore (_)
142    Underscore,
143    /// Backtick (`)
144    Backtick,
145    /// Tilde (~)
146    Tilde,
147    /// Hash marker (#)
148    Hash,
149    /// Pipe separator (|)
150    Pipe,
151    /// Dash or hyphen (-)
152    Dash,
153    /// Plus sign (+)
154    Plus,
155    /// Dot or period (.)
156    Dot,
157    /// Colon (:)
158    Colon,
159    /// Exclamation mark (!)
160    Exclamation,
161
162    /// Error token
163    Error,
164
165    /// Root node
166    Root,
167    /// Document node
168    Document,
169    /// Paragraph node
170    Paragraph,
171    /// Metadata section
172    Metadata,
173
174    /// End of file
175    Eof,
176}
oak_notedown/lexer/token_type.rs

oak_notedown/lexer/
token_type.rs
