regen/sdk/token/
mod.rs

1use html_escape;
2
3mod token_type;
4pub use token_type::TokenType;
5mod token_stream;
6pub use token_stream::TokenStream;
7mod token_blocks;
8pub use token_blocks::TokenBlocks;
9pub mod lex;
10
11/// Generic Token implementation
12///
13/// A Token is anything that contains a type, a value, and a position in the input text
14#[derive(PartialEq, Default, Debug, Clone)]
15pub struct TokenImpl<T>
16where
17    T: TokenType,
18{
19    /// Token type
20    pub token_type: T,
21    /// Token value
22    pub value: String,
23    /// Token absolute position in the input text
24    /// (start, end) - start is inclusive, end is exclusive
25    pub pos: (usize, usize),
26}
27
28impl<T> TokenImpl<T>
29where
30    T: TokenType,
31{
32    /// Convert to another token type, with the value and position untouched
33    #[inline]
34    pub fn into<TO>(self) -> TokenImpl<TO>
35    where
36        TO: From<T> + TokenType,
37    {
38        TokenImpl {
39            token_type: self.token_type.into(),
40            value: self.value,
41            pos: self.pos,
42        }
43    }
44    #[inline]
45    /// Convert the token to a raw html-escaped text without applying any style.
46    pub fn to_html_text(&self) -> String {
47        html_escape::encode_text(&self.value).to_string()
48    }
49
50    /// Convert the token to an html string with `class` attributes provided by the token type (may be `None`)
51    ///
52    /// This may return a `<span>` tag, or a raw string if the class name is `None`.
53    pub fn to_html(&self) -> String {
54        let text = self.to_html_text();
55        match self.token_type.html_class() {
56            None => text,
57            Some(class_name) => {
58                format!(r#"<span class="{class_name}">{text}</span>"#)
59            }
60        }
61    }
62}
regen/sdk/token/mod.rs

regen/sdk/token/
mod.rs
