jayce 12.1.0

jayce is a tokenizer 🌌
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

pub mod internal;
use regex::Regex;

pub struct Duo<T> {
    pub kind: T,
    pub regex: Regex,
    pub preserve: bool,
}

impl<T> Duo<T> {
    pub fn new(kind: T, regex: &str, preserve: bool) -> Self {
        Self {
            kind,
            regex: Regex::new(regex).unwrap(),
            preserve,
        }
    }
}

pub struct Tokenizer<'a, T> {
    source: &'a str,
    duos: &'a [Duo<T>],
    pub cursor: usize,
    pub line: usize,
    pub column: usize,
    next: Option<Token<'a, T>>,
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Token<'a, T> {
    pub kind: &'a T,
    pub value: &'a str,
    pub pos: (usize, usize),
}

impl<'a, T> Tokenizer<'a, T> {
    #[inline]
    pub fn new(source: &'a str, duos: &'a [Duo<T>]) -> Self {
        Self {
            source,
            duos,
            cursor: 0,
            line: 1,
            column: 1,
            next: None,
        }
    }

    fn advance(&mut self) -> Result<Option<Token<'a, T>>, Box<dyn std::error::Error>> {
        while self.cursor < self.source.len() {
            let mut matched = false;

            for duo in self.duos.iter() {
                if let Some(result) = duo.regex.find(&self.source[self.cursor..]) {
                    let value: &str = result.as_str();
                    let token_pos = (self.line, self.column);
                    let len = result.len();
                    self.cursor += len;
                    let newlines_count = bytecount::count(value.as_bytes(), b'\n');
                    if newlines_count > 0 {
                        self.line += newlines_count;
                        self.column = len - value.rfind('\n').unwrap_or(1);
                    } else {
                        self.column += len;
                    }

                    if duo.preserve {
                        return Ok(Some(Token {
                            kind: &duo.kind,
                            value,
                            pos: token_pos,
                        }));
                    } else {
                        matched = true;
                        break;
                    }
                }
            }

            if !matched {
                return Err(format!(
                    "Failed to match at line {}, column {}.",
                    self.line, self.column
                ))?;
            }
        }

        Ok(None)
    }

    pub fn peek(&mut self) -> Result<Option<Token<'a, T>>, Box<dyn std::error::Error>>
    where
        T: Clone,
    {
        if self.next.is_none() {
            self.next = self.advance()?;
        }

        Ok(self.next.clone())
    }

    pub fn consume(&mut self) -> Result<Option<Token<'a, T>>, Box<dyn std::error::Error>> {
        if self.next.is_none() {
            self.next = self.advance()?;
        }

        let result = Ok(self.next.take());
        self.next = self.advance()?;
        result
    }

    pub fn consume_all(&mut self) -> Result<Vec<Token<'a, T>>, Box<dyn std::error::Error>> {
        let mut tokens: Vec<Token<'_, T>> = Vec::new();
        while let Some(token) = self.consume()? {
            tokens.push(token);
        }
        Ok(tokens)
    }
}