1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
use unicode_width::UnicodeWidthChar;

/// For storing tokens to put into a string
/// It has a start token, to mark the start of a token
/// It has a text token, for the text inbetween and inside tokens
/// It also has an end token, to mark the end of a token
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum Token {
    Start(String),
    Text(String),
    End(String),
}

#[derive(Debug, PartialEq, Eq, Clone)]
pub enum TokOpt {
    Some(String, String),
    None(String),
}

impl TokOpt {
    pub fn is_empty(&self) -> bool {
        match self {
            TokOpt::Some(text, _) => text.len() == 0,
            TokOpt::None(text) => text.len() == 0,
        }
    }

    pub fn nibble(&mut self) -> Option<char> {
        match self {
            TokOpt::Some(ref mut text, _) => {
                let ch = *text.chars().collect::<Vec<_>>().get(0)?;
                text.remove(0);
                if UnicodeWidthChar::width(ch)? > 1 {
                    text.insert(0, ' ');
                }
                Some(ch)
            }
            TokOpt::None(ref mut text) => {
                let ch = *text.chars().collect::<Vec<_>>().get(0)?;
                text.remove(0);
                if UnicodeWidthChar::width(ch)? > 1 {
                    text.insert(0, ' ');
                }
                Some(ch)
            }
        }
    }
}

/// For storing all the data in a token to prevent overwriting
/// This contains the contents, type, start and end of the token
/// This is used to compare tokens to each other to prevent tokens inside tokens
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct FullToken {
    pub text: String,
    pub kind: String,
    pub start: usize,
    pub end: usize,
    pub multi: bool,
}

impl FullToken {
    /// Returns the length of the token
    pub fn len(&self) -> usize {
        self.text.len()
    }

    /// Determines if the token is empty
    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }
}