weavatrix_parse/token/
construction.rs

1use super::{Language, Mode, Token, TokenKind, Tokenizer};
2
3impl<'source> Tokenizer<'source> {
4    #[must_use]
5    pub fn new(source: &'source str, language: Language) -> Self {
6        Self {
7            source,
8            syntax: language.syntax(),
9            mode: Mode::Lossless,
10            bytes: source.as_bytes(),
11            offset: 0,
12            line: 1,
13            column: 1,
14            value_before: false,
15            at_line_start: true,
16        }
17    }
18
19    /// Selects how much of the source the stream carries.
20    #[must_use]
21    pub const fn mode(mut self, mode: Mode) -> Self {
22        self.mode = mode;
23        self
24    }
25
26    pub(super) fn peek(&self, ahead: usize) -> Option<u8> {
27        self.bytes.get(self.offset + ahead).copied()
28    }
29
30    pub(super) fn starts_with(&self, needle: &str) -> bool {
31        self.source[self.offset..].starts_with(needle)
32    }
33
34    /// Advances past `count` bytes, keeping the line and column current.
35    pub(super) fn advance(&mut self, count: usize) {
36        let end = (self.offset + count).min(self.bytes.len());
37        while self.offset < end {
38            let character = self.source[self.offset..].chars().next().unwrap_or('\0');
39            if character == '\n' {
40                self.line += 1;
41                self.column = 1;
42            } else {
43                self.column += 1;
44            }
45            self.offset += character.len_utf8();
46        }
47    }
48
49    pub(super) fn emit(&mut self, kind: TokenKind, start: usize, line: u32, column: u32) -> Token {
50        Token {
51            kind,
52            start,
53            end: self.offset,
54            line,
55            column,
56        }
57    }
58}
weavatrix_parse/token/construction.rs

weavatrix_parse/token/
construction.rs
