use std::borrow::Cow;
use crate::Span;
use crate::language::LanguageError;
#[derive(Clone, Debug)]
pub struct Scanner<'a> {
bytes: &'a [u8],
pub pos: usize,
}
impl<'a> Scanner<'a> {
pub fn new(bytes: &'a [u8]) -> Self {
Scanner { bytes, pos: 0 }
}
pub fn bytes(&self) -> &'a [u8] {
self.bytes
}
pub fn len(&self) -> usize {
self.bytes.len()
}
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
pub fn eof(&self) -> bool {
self.pos >= self.bytes.len()
}
pub fn byte(&self, k: isize) -> Option<u8> {
let i = self.pos.checked_add_signed(k)?;
self.bytes.get(i).copied()
}
pub fn starts(&self, s: &str) -> bool {
self.bytes[self.pos.min(self.bytes.len())..].starts_with(s.as_bytes())
}
pub fn slice(&self, s: usize, e: usize) -> Cow<'a, str> {
String::from_utf8_lossy(&self.bytes[s..e])
}
pub fn lit(&mut self, s: &str) -> Option<Span> {
if !self.starts(s) {
return None;
}
let start = self.pos;
self.pos += s.len();
Some(Span::from(start..self.pos))
}
pub fn take_while(&mut self, f: impl Fn(u8) -> bool) -> Span {
let start = self.pos;
while self.pos < self.bytes.len() && f(self.bytes[self.pos]) {
self.pos += 1;
}
Span::from(start..self.pos)
}
pub fn take_while1(&mut self, f: impl Fn(u8) -> bool) -> Option<Span> {
let s = self.take_while(f);
if s.start == s.end { None } else { Some(s) }
}
pub fn find(&self, s: &str) -> Option<usize> {
let needle = s.as_bytes();
if needle.is_empty() {
return Some(self.pos);
}
let hay = self.bytes.get(self.pos..)?;
hay.windows(needle.len()).position(|w| w == needle).map(|i| self.pos + i)
}
pub fn advance(&mut self, k: usize) -> usize {
self.pos += k;
self.pos
}
pub fn hs(&mut self) -> usize {
self.take_while(|b| b == b' ' || b == b'\t');
self.pos
}
pub fn same_line(&self, a: usize, b: usize) -> bool {
let b = b.min(self.bytes.len());
a >= b || !self.bytes[a..b].contains(&b'\n')
}
pub fn fail(&self, message: impl Into<String>) -> LanguageError {
LanguageError::at(message, self.pos)
}
pub fn fail_at(&self, message: impl Into<String>, at: usize) -> LanguageError {
LanguageError::at(message, at)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Token<K> {
pub kind: K,
pub span: Span,
}
#[derive(Clone, Debug)]
pub struct Cursor<'t, K> {
tokens: &'t [Token<K>],
pub pos: usize,
}
impl<'t, K: Copy + PartialEq> Cursor<'t, K> {
pub fn new(tokens: &'t [Token<K>]) -> Self {
assert!(!tokens.is_empty(), "a cursor needs an end-of-input token");
Cursor { tokens, pos: 0 }
}
pub fn peek(&self) -> Token<K> {
self.tokens[self.pos]
}
pub fn ahead(&self, k: usize) -> Token<K> {
self.tokens[(self.pos + k).min(self.tokens.len() - 1)]
}
pub fn advance(&mut self) -> Token<K> {
let t = self.tokens[self.pos];
if self.pos < self.tokens.len() - 1 {
self.pos += 1;
}
t
}
pub fn at(&self, kind: K) -> bool {
self.tokens[self.pos].kind == kind
}
pub fn take(&mut self, kind: K) -> Option<Token<K>> {
if self.at(kind) { Some(self.advance()) } else { None }
}
pub fn fail(&self, message: impl Into<String>) -> LanguageError {
LanguageError::at(message, self.tokens[self.pos].span.start)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scanner() {
let mut sc = Scanner::new("é # c\nx".as_bytes());
assert_eq!(sc.byte(-1), None);
assert_eq!(sc.byte(0), Some(0xc3));
assert_eq!(sc.find("#"), Some(3));
assert!(sc.same_line(0, 6));
assert!(!sc.same_line(0, 7));
assert_eq!(sc.take_while1(|b| b >= 0x80), Some(Span::from(0..2)));
assert_eq!(sc.lit("#"), None);
assert_eq!(sc.fail("x").byte_offset, Some(2));
}
#[test]
fn cursor() {
let toks = [
Token { kind: 'a', span: Span::from(0..1) },
Token { kind: '$', span: Span::from(1..1) },
];
let mut c = Cursor::new(&toks);
assert!(c.take('b').is_none());
assert_eq!(c.take('a').unwrap().span, Span::from(0..1));
c.advance();
assert!(c.at('$'));
assert_eq!(c.ahead(3).kind, '$');
}
}