hemtt_tokens/
symbol.rs

1use crate::whitespace::Whitespace;
2
3#[derive(Clone, Debug, PartialEq, Eq)]
4/// The symbol of a [`Token`](crate::Token)
5pub enum Symbol {
6    /// A word is a contiguous sequence of letters, digits, and underscores.
7    /// A word will never start with a digit.
8    Word(String),
9    /// A single alphanumeric character
10    Alpha(char),
11    /// Parsed digits will always be a single digit, but generated digits may have multiple digits.
12    Digit(usize),
13    /// _
14    Underscore,
15    /// -
16    Dash,
17    /// =
18    Assignment,
19    /// +
20    Plus,
21    /// {
22    LeftBrace,
23    /// }
24    RightBrace,
25    /// [
26    LeftBracket,
27    /// ]
28    RightBracket,
29    /// (
30    LeftParenthesis,
31    /// )
32    RightParenthesis,
33    /// :
34    Colon,
35    /// ;
36    Semicolon,
37    /// ##
38    Join,
39    /// #
40    Directive,
41    /// \
42    Escape,
43    /// /
44    Slash,
45    /// ,
46    Comma,
47    /// .
48    Decimal,
49    /// ""
50    DoubleQuote,
51    /// '
52    SingleQuote,
53    /// <
54    LeftAngle,
55    /// >
56    RightAngle,
57
58    /// A unicode character
59    Unicode(String),
60
61    /// A newline \n
62    Newline,
63    /// A [`Whitespace`] character
64    Whitespace(Whitespace),
65    /// A comment
66    /// Comments are not parsed, but are kept in the token stream
67    /// so that they can be outputted in the same format as the input.
68    ///
69    /// Comments have two forms:
70    /// Single line comments start with `//` and end with a newline.
71    /// Multi line comments start with `/*` and end with `*/`.
72    Comment(String),
73
74    /// End of input
75    Eoi,
76    /// Void token, not outputted
77    Void,
78}
79
80impl Symbol {
81    /// Create a new [`Word`](Symbol::Word) symbol
82    pub fn from_word<S: Into<String>>(word: S) -> Self {
83        Self::Word(word.into())
84    }
85
86    #[must_use]
87    /// Check if a symbol is [`Whitespace`](Symbol::Whitespace) or [`Comment`](Symbol::Comment)
88    pub const fn is_whitespace(&self) -> bool {
89        matches!(&self, Self::Whitespace(_) | Self::Comment(_))
90    }
91
92    #[must_use]
93    /// Check if a symbol is [`Newline`](Symbol::Newline)
94    pub const fn is_newline(&self) -> bool {
95        matches!(&self, Self::Newline)
96    }
97
98    #[must_use]
99    /// Get the output of a symbol
100    pub fn output(&self) -> String {
101        match *self {
102            Self::Join => String::new(),
103            _ => self.to_string(),
104        }
105    }
106}
107
108impl ToString for Symbol {
109    fn to_string(&self) -> String {
110        match self {
111            Self::Word(w) => w.clone(),
112            Self::Alpha(c) => c.to_string(),
113            Self::Digit(d) => d.to_string(),
114            Self::Underscore => "_".to_string(),
115            Self::Dash => "-".to_string(),
116            Self::Assignment => "=".to_string(),
117            Self::Plus => "+".to_string(),
118            Self::LeftBrace => "{".to_string(),
119            Self::RightBrace => "}".to_string(),
120            Self::LeftBracket => "[".to_string(),
121            Self::RightBracket => "]".to_string(),
122            Self::LeftParenthesis => "(".to_string(),
123            Self::RightParenthesis => ")".to_string(),
124            Self::Colon => ":".to_string(),
125            Self::Semicolon => ";".to_string(),
126            Self::Join => "##".to_string(),
127            Self::Directive => "#".to_string(),
128            Self::Escape => "\\".to_string(),
129            Self::Slash => "/".to_string(),
130            Self::Comma => ",".to_string(),
131            Self::Decimal => ".".to_string(),
132            Self::DoubleQuote => "\"".to_string(),
133            Self::SingleQuote => "'".to_string(),
134            Self::LeftAngle => "<".to_string(),
135            Self::RightAngle => ">".to_string(),
136            Self::Unicode(s) => s.to_string(),
137            Self::Newline => "\n".to_string(),
138            Self::Whitespace(w) => w.to_string(),
139            Self::Eoi | Self::Void | Self::Comment(_) => String::new(),
140        }
141    }
142}
hemtt_tokens/symbol.rs

hemtt_tokens/
symbol.rs
