use super::{Lexer, LexerPos};
#[inline]
fn bump_line_col(line: &mut usize, column: &mut usize, slice: &str) {
match memchr::memrchr(b'\n', slice.as_bytes()) {
None => {
*column += if slice.is_ascii() {
slice.len()
} else {
slice.chars().count()
};
}
Some(last_nl) => {
*line += memchr::memchr_iter(b'\n', slice.as_bytes()).count();
let tail = &slice[last_nl + 1..];
*column = if tail.is_ascii() {
tail.len()
} else {
tail.chars().count()
};
}
}
}
impl Lexer {
#[inline]
pub(super) fn rest(&self) -> &str {
&self.source[self.byte_pos..]
}
#[inline]
pub(crate) fn peek_byte(&self) -> Option<u8> {
self.source.as_bytes().get(self.byte_pos).copied()
}
#[inline]
pub(crate) fn peek(&self) -> Option<char> {
let b = self.peek_byte()?;
if b < 0x80 {
Some(b as char)
} else {
self.rest().chars().next()
}
}
#[inline]
pub(crate) fn peek_ahead(&self, n: usize) -> Option<char> {
self.rest().chars().nth(n)
}
#[inline]
pub(crate) fn at(&self, s: &str) -> bool {
self.source.as_bytes()[self.byte_pos..].starts_with(s.as_bytes())
}
#[inline]
pub(crate) fn advance_by(&mut self, n: usize) {
for _ in 0..n {
self.advance();
}
}
#[inline]
pub(super) const fn mark(&self) -> LexerPos {
LexerPos {
byte_pos: self.byte_pos,
line: self.line,
column: self.column,
}
}
#[inline]
pub(super) const fn reset(&mut self, mark: LexerPos) {
self.byte_pos = mark.byte_pos;
self.line = mark.line;
self.column = mark.column;
}
#[inline]
pub(super) fn try_with_cursor<T>(
&mut self,
f: impl FnOnce(&mut Self) -> Option<T>,
) -> Option<T> {
let mark = self.mark();
let r = f(self);
if r.is_none() {
self.reset(mark);
}
r
}
#[inline]
pub(crate) fn advance(&mut self) -> Option<char> {
let b = self.peek_byte()?;
if b < 0x80 {
self.byte_pos += 1;
if b == b'\n' {
self.line += 1;
self.column = 0;
} else {
self.column += 1;
}
Some(b as char)
} else {
let ch = self.rest().chars().next()?;
self.byte_pos += ch.len_utf8();
self.column += 1;
Some(ch)
}
}
#[inline]
pub(crate) fn scan_until3(&mut self, a: u8, b: u8, c: u8) -> &str {
let rest = &self.source.as_bytes()[self.byte_pos..];
let len = memchr::memchr3(a, b, c, rest).unwrap_or(rest.len());
if len == 0 {
return "";
}
let start = self.byte_pos;
let end = start + len;
self.byte_pos = end;
bump_line_col(&mut self.line, &mut self.column, &self.source[start..end]);
&self.source[start..end]
}
pub(super) fn seek_to(&mut self, target: usize) {
debug_assert!(target >= self.byte_pos);
let start = self.byte_pos;
self.byte_pos = target;
bump_line_col(
&mut self.line,
&mut self.column,
&self.source[start..target],
);
}
#[inline]
pub(super) fn advance_bytes_no_newline(&mut self, len: usize) -> &str {
let start = self.byte_pos;
let end = start + len;
debug_assert!(!self.source.as_bytes()[start..end].contains(&b'\n'));
self.byte_pos = end;
bump_line_col(&mut self.line, &mut self.column, &self.source[start..end]);
&self.source[start..end]
}
#[inline]
pub(super) fn take_ascii_while(&mut self, pred: impl Fn(u8) -> bool) -> &str {
let bytes = self.source.as_bytes();
let start = self.byte_pos;
let mut i = start;
while i < bytes.len() && pred(bytes[i]) {
i += 1;
}
self.byte_pos = i;
self.column += i - start;
&self.source[start..i]
}
#[inline]
pub(super) fn is_eof(&self) -> bool {
self.byte_pos >= self.source.len()
}
}