use core::str;
use std::borrow::Cow;
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Token<'a> {
Word(&'a str),
Quoted(Quoted<'a>),
Open,
Close,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Quoted<'a> {
Borrowed(&'a [u8]),
Owned(Vec<u8>),
}
impl<'a> Quoted<'a> {
pub(crate) fn into_cow(self) -> Cow<'a, [u8]> {
match self {
Self::Borrowed(bytes) => Cow::Borrowed(bytes),
Self::Owned(bytes) => Cow::Owned(bytes),
}
}
pub(crate) fn into_ident(
self,
offset: usize,
) -> Result<Ident<'a>, LexError> {
match self {
Self::Borrowed(bytes) => str::from_utf8(bytes)
.map(|text| Ident(Cow::Borrowed(text)))
.map_err(|_| LexError::NotUtf8 { offset }),
Self::Owned(bytes) => String::from_utf8(bytes)
.map(|text| Ident(Cow::Owned(text)))
.map_err(|_| LexError::NotUtf8 { offset }),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Ident<'a>(Cow<'a, str>);
impl<'a> Ident<'a> {
pub(crate) fn as_str(&self) -> &str {
&self.0
}
pub(crate) fn into_cow(self) -> Cow<'a, str> {
self.0
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum LexError {
UnterminatedString { offset: usize },
NotUtf8 { offset: usize },
BadEscape { offset: usize },
}
pub(crate) struct Lexer<'a> {
input: &'a [u8],
position: usize,
token_start: usize,
}
impl<'a> Lexer<'a> {
pub(crate) const fn new(input: &'a [u8]) -> Self {
Self {
input,
position: 0,
token_start: 0,
}
}
#[cfg(feature = "parallel")]
pub(crate) const fn starting_at(input: &'a [u8], position: usize) -> Self {
Self {
input,
position,
token_start: position,
}
}
pub(crate) const fn offset(&self) -> usize {
self.token_start
}
fn skip_trivia(&mut self) {
loop {
while self
.input
.get(self.position)
.is_some_and(u8::is_ascii_whitespace)
{
self.position += 1;
}
if self.input.get(self.position) == Some(&b'#') {
self.position =
memchr::memchr(b'\n', &self.input[self.position..])
.map_or(self.input.len(), |end| self.position + end);
} else {
break;
}
}
}
pub(crate) fn next_token(&mut self) -> Result<Option<Token<'a>>, LexError> {
self.skip_trivia();
self.token_start = self.position;
let Some(&byte) = self.input.get(self.position) else {
return Ok(None);
};
match byte {
b'[' => {
self.position += 1;
Ok(Some(Token::Open))
}
b']' => {
self.position += 1;
Ok(Some(Token::Close))
}
b'"' => self.string().map(Some),
_ => self.word().map(Some),
}
}
fn word(&mut self) -> Result<Token<'a>, LexError> {
let start = self.position;
while let Some(&byte) = self.input.get(self.position) {
if byte.is_ascii_whitespace()
|| matches!(byte, b'[' | b']' | b'"' | b'#')
{
break;
}
self.position += 1;
}
str::from_utf8(&self.input[start..self.position])
.map(Token::Word)
.map_err(|_| LexError::NotUtf8 { offset: start })
}
fn string(&mut self) -> Result<Token<'a>, LexError> {
let open = self.position;
self.position += 1;
let start = self.position;
let mut escaped = false;
loop {
let Some(&byte) = self.input.get(self.position) else {
return Err(LexError::UnterminatedString { offset: open });
};
match byte {
b'\\' => {
escaped = true;
self.position += 2;
}
b'"' => break,
_ => self.position += 1,
}
}
let raw = &self.input[start..self.position];
self.position += 1;
if escaped {
unescape(raw, start)
.map(|bytes| Token::Quoted(Quoted::Owned(bytes)))
} else {
Ok(Token::Quoted(Quoted::Borrowed(raw)))
}
}
}
pub(crate) fn unescape(raw: &[u8], offset: usize) -> Result<Vec<u8>, LexError> {
let mut out = Vec::with_capacity(raw.len());
let mut index = 0;
while index < raw.len() {
if raw[index] == b'\\' {
let Some(&escape) = raw.get(index + 1) else {
return Err(LexError::BadEscape {
offset: offset + index,
});
};
match escape {
b'n' => out.push(b'\n'),
b'r' => out.push(b'\r'),
b't' => out.push(b'\t'),
b'"' => out.push(b'"'),
b'\\' => out.push(b'\\'),
b'0'..=b'7' => {
let digits = raw[index + 1..]
.iter()
.take(3)
.take_while(|d| (b'0'..=b'7').contains(d))
.count();
let value = raw[index + 1..index + 1 + digits]
.iter()
.fold(0u32, |acc, d| acc * 8 + u32::from(d - b'0'));
out.push(u8::try_from(value).map_err(|_| {
LexError::BadEscape {
offset: offset + index,
}
})?);
index += digits - 1;
}
_ => {
return Err(LexError::BadEscape {
offset: offset + index,
});
}
}
index += 2;
} else {
out.push(raw[index]);
index += 1;
}
}
Ok(out)
}