oak_markdown/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// Represents a token in a Markdown document.
4pub type MarkdownToken = Token<MarkdownTokenType>;
5
6/// Token types for the Markdown language.
7#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
8#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
9pub enum MarkdownTokenType {
10    /// Plain text.
11    Text,
12    /// Whitespace characters.
13    Whitespace,
14    /// A newline character.
15    Newline,
16    /// Heading level 1: `#`.
17    Heading1,
18    /// Heading level 2: `##`.
19    Heading2,
20    /// Heading level 3: `###`.
21    Heading3,
22    /// Heading level 4: `####`.
23    Heading4,
24    /// Heading level 5: `#####`.
25    Heading5,
26    /// Heading level 6: `######`.
27    Heading6,
28    /// The text content of a heading.
29    HeadingText,
30    /// Emphasized text: `*` or `_`.
31    Emphasis,
32    /// Strong text: `**` or `__`.
33    Strong,
34    /// Strikethrough text: `~~`.
35    Strikethrough,
36    /// Inline code: `` ` ``.
37    InlineCode,
38    /// A code block.
39    CodeBlock,
40    /// A code fence: ` ``` ` or `~~~`.
41    CodeFence,
42    /// The language identifier for a code block.
43    CodeLanguage,
44    /// A link.
45    Link,
46    /// The text of a link.
47    LinkText,
48    /// The URL of a link.
49    LinkUrl,
50    /// The title of a link.
51    LinkTitle,
52    /// An image.
53    Image,
54    /// The alt text of an image.
55    ImageAlt,
56    /// The URL of an image.
57    ImageUrl,
58    /// The title of an image.
59    ImageTitle,
60    /// A blockquote.
61    Blockquote,
62    /// The marker for a blockquote: `>`.
63    BlockquoteMarker,
64    /// A list.
65    List,
66    /// An item in a list.
67    ListItem,
68    /// The marker for a list item: `-`, `*`, `+`, or `1.`.
69    ListMarker,
70    /// An unordered list.
71    UnorderedList,
72    /// An ordered list.
73    OrderedList,
74    /// A task list.
75    TaskList,
76    /// The marker for a task: `[ ]` or `[x]`.
77    TaskMarker,
78    /// A table.
79    Table,
80    /// A row in a table.
81    TableRow,
82    /// A cell in a table.
83    TableCell,
84    /// A table header cell.
85    TableHeader,
86    /// A table separator row.
87    TableSeparator,
88    /// Table column alignment.
89    TableAlignment,
90    /// A horizontal rule: `---`, `***`, or `___`.
91    HorizontalRule,
92    /// Inline math: `$`.
93    MathInline,
94    /// Block math: `$$`.
95    MathBlock,
96    /// Front matter: `---`.
97    FrontMatter,
98    /// A footnote definition.
99    FootnoteDefinition,
100    /// A footnote reference.
101    FootnoteReference,
102    /// A definition list.
103    DefinitionList,
104    /// A term in a definition list.
105    DefinitionTerm,
106    /// A description in a definition list.
107    DefinitionDescription,
108    /// Superscript text.
109    Superscript,
110    /// Subscript text.
111    Subscript,
112    /// An abbreviation.
113    Abbreviation,
114    /// An HTML tag.
115    HtmlTag,
116    /// An HTML comment.
117    HtmlComment,
118    /// An XML tag.
119    XmlTag,
120    /// An XML comment.
121    XmlComment,
122    /// An asterisk: `*`.
123    Asterisk,
124    /// An underscore: `_`.
125    Underscore,
126    /// A backtick: `` ` ``.
127    Backtick,
128    /// A tilde: `~`.
129    Tilde,
130    /// A hash sign: `#`.
131    Hash,
132    /// A less than sign: `<`.
133    Less,
134    /// A greater than sign: `>`.
135    Greater,
136    /// A left bracket: `[`.
137    LBracket,
138    /// A right bracket: `]`.
139    RBracket,
140    /// A left parenthesis: `(`.
141    LParen,
142    /// A right parenthesis: `)`.
143    RParen,
144    /// An exclamation mark: `!`.
145    Exclamation,
146    /// A pipe symbol: `|`.
147    Pipe,
148    /// A dash: `-`.
149    Dash,
150    /// A plus sign: `+`.
151    Plus,
152    /// A dot: `.`.
153    Dot,
154    /// A colon: `:`.
155    Colon,
156    /// A dollar sign: `$`.
157    Dollar,
158    /// A caret symbol: `^`.
159    Caret,
160    /// An escape character: `\`.
161    Escape,
162    /// An error token.
163    Error,
164    /// The root node of a Markdown document.
165    Root,
166    /// A Markdown document.
167    Document,
168    /// A paragraph of text.
169    Paragraph,
170    /// An automatic link (HTTP/HTTPS URL).
171    AutoLink,
172    /// End of stream.
173    EndOfStream,
174}
175
176impl TokenType for MarkdownTokenType {
177    type Role = UniversalTokenRole;
178    const END_OF_STREAM: Self = Self::EndOfStream;
179
180    fn is_ignored(&self) -> bool {
181        false
182    }
183
184    fn role(&self) -> Self::Role {
185        match self {
186            _ => UniversalTokenRole::None,
187        }
188    }
189}
oak_markdown/lexer/token_type.rs

oak_markdown/lexer/
token_type.rs
