note_mark/model/
token.rs

1//! Token.
2
3/// The struct to represent a token.
4///
5/// Token contains the kind of token and the range of the token in the source.
6/// The range is represented by the start position and the length of the token.
7///
8/// Note!: **This struct can live longer than the source string.**
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub struct Token {
11    /// The kind of the token.
12    pub kind: TokenKind,
13    /// The start position of the token.
14    pub start: usize,
15    /// The length of the token.
16    pub len: usize,
17}
18
19impl Token {
20    /// Get the range of the token.
21    pub fn range(&self) -> std::ops::Range<usize> {
22        self.start..self.start + self.len
23    }
24}
25
26/// The enum to represent a token kind.
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum TokenKind {
29    Text,
30    /// " "
31    Space,
32    /// "\t"
33    Tab,
34    Break,
35    /// "\\"
36    Backslash,
37    /// "#"
38    Pound,
39    /// "*"
40    Star,
41    /// ":"
42    Colon,
43    /// "`"
44    Backquote,
45    /// ">"
46    Gt,
47    /// "-"
48    Hyphen,
49    /// "|"
50    VerticalBar,
51    /// "."
52    Dot,
53    /// "("
54    OpenParen,
55    /// ")"
56    CloseParen,
57    /// "{"
58    OpenBrace,
59    /// "}"
60    CloseBrace,
61    /// "["
62    OpenBracket,
63    /// "]"
64    CloseBracket,
65}
note_mark/model/token.rs

note_mark/model/
token.rs
