oak_core/lexer/
scan_white_space.rs

1use super::LexerState;
2use crate::{
3    Language,
4    source::{SimdScanner, Source},
5};
6
7/// Configuration for whitespace scanning.
8pub struct WhitespaceConfig {
9    /// Whether to recognize Unicode whitespace characters.
10    pub unicode_whitespace: bool,
11}
12
13impl WhitespaceConfig {
14    /// Scans for whitespace at the current position in the lexer state.
15    pub fn scan<S: Source + ?Sized, L: Language>(&self, state: &mut LexerState<S, L>, kind: L::TokenType) -> bool {
16        let start = state.get_position();
17        let range = if self.unicode_whitespace { state.take_while(|c| c.is_whitespace()) } else { state.skip_ascii_whitespace() };
18
19        if range.end > start {
20            state.add_token(kind, start, range.end);
21            true
22        }
23        else {
24            false
25        }
26    }
27}
28
29/// Counts how many spaces or tabs are at the start of the byte slice.
30#[inline]
31#[allow(dead_code)]
32pub fn count_space_tab_prefix(bytes: &[u8]) -> usize {
33    SimdScanner::skip_two_bytes(bytes, b' ', b'\t')
34}
oak_core/lexer/scan_white_space.rs

oak_core/lexer/
scan_white_space.rs
