use std::ops::Range;
use rhai::{Position, Token, TokenizeState};
use crate::stream::{LineIndex, StrStream};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct At {
pub line: u32,
pub column: u32,
}
impl At {
pub(crate) fn new(lines: &LineIndex, src: &str, offset: usize) -> Self {
let (line, column) = lines.line_col_of(src, offset);
Self {
line: line + 1,
column: column + 1,
}
}
}
impl std::fmt::Display for At {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "line {}, column {}", self.line, self.column)
}
}
#[derive(Debug, Clone)]
pub struct Tok {
pub token: Token,
pub span: Range<usize>,
}
impl Tok {
pub fn text<'s>(&self, src: &'s str) -> &'s str {
&src[self.span.clone()]
}
pub fn is_comment(&self) -> bool {
matches!(self.token, Token::Comment(..))
}
}
#[derive(Debug)]
pub enum LexFailure {
Lex { message: String, at: At },
OverlappingSpan {
at: At,
span: Range<usize>,
previous_end: usize,
},
NonWhitespaceGap { at: At, gap: String },
NonWhitespaceTail { tail: String },
UnterminatedInterpolation,
}
impl std::fmt::Display for LexFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Lex { message, at } => write!(f, "{message} ({at})"),
Self::OverlappingSpan {
at,
span,
previous_end,
} => write!(
f,
"span {span:?} at {at} overlaps the previous token, which ended at {previous_end}"
),
Self::NonWhitespaceGap { at, gap } => {
write!(f, "non-whitespace {gap:?} precedes the token at {at}")
}
Self::NonWhitespaceTail { tail } => {
write!(f, "non-whitespace {tail:?} follows the last token")
}
Self::UnterminatedInterpolation => {
f.write_str("input ended with an interpolated string still open")
}
}
}
}
impl std::error::Error for LexFailure {}
fn track_interpolation(token: &Token, frames: &mut Vec<usize>, state: &mut TokenizeState) {
match token {
Token::InterpolatedString(..) => frames.push(0),
Token::LeftBrace | Token::MapStart => {
if let Some(depth) = frames.last_mut() {
*depth += 1;
}
}
Token::RightBrace => {
if let Some(depth) = frames.last_mut() {
*depth -= 1;
if *depth == 0 {
frames.pop();
state.is_within_text_terminated_by = Some("`".into());
}
}
}
_ => (),
}
}
fn skip_trivia(src: &str, mut at: usize) -> (usize, Vec<Range<usize>>) {
let mut swallowed = Vec::new();
loop {
at += src[at..]
.bytes()
.take_while(u8::is_ascii_whitespace)
.count();
if src[at..].starts_with("//!") {
let end = src[at..]
.find('\n')
.map_or(src.len(), |offset| at + offset + 1);
swallowed.push(at..end);
at = end;
continue;
}
return (at, swallowed);
}
}
fn accept(
src: &str,
lines: &LineIndex,
out: &mut Vec<Tok>,
tiled: &mut usize,
token: Token,
span: Range<usize>,
) -> Result<(), LexFailure> {
if span.start < *tiled {
return Err(LexFailure::OverlappingSpan {
at: At::new(lines, src, span.start),
span,
previous_end: *tiled,
});
}
let gap = &src[*tiled..span.start];
if !gap.chars().all(char::is_whitespace) {
return Err(LexFailure::NonWhitespaceGap {
at: At::new(lines, src, span.start),
gap: gap.to_string(),
});
}
*tiled = span.end;
out.push(Tok { token, span });
Ok(())
}
pub fn tokenize(src: &str) -> Result<Vec<Tok>, LexFailure> {
let lines = LineIndex::new(src);
let mut stream = StrStream::new(src);
let mut state = TokenizeState {
include_comments: true,
..Default::default()
};
let mut pos = Position::START;
let mut out = Vec::new();
let mut tiled = 0usize;
let mut interpolations: Vec<usize> = Vec::new();
loop {
let resuming_text = state.is_within_text_terminated_by.is_some();
let before = stream.byte_offset();
let unary_before = state.next_token_cannot_be_unary;
let (token, _) = rhai::get_next_token(&mut stream, &mut state, &mut pos);
if let Token::Comment(..) = token {
state.next_token_cannot_be_unary = unary_before;
}
let end = stream.byte_offset();
let start = if resuming_text {
before
} else {
let (start, swallowed) = skip_trivia(src, before);
for span in swallowed {
let text = src[span.clone()].to_string();
accept(
src,
&lines,
&mut out,
&mut tiled,
Token::Comment(Box::new(text)),
span,
)?;
}
start
};
if let Token::EOF = token {
break;
}
if let Token::LexError(err) = &token {
return Err(LexFailure::Lex {
message: err.to_string(),
at: At::new(&lines, src, start),
});
}
track_interpolation(&token, &mut interpolations, &mut state);
accept(src, &lines, &mut out, &mut tiled, token, start..end)?;
}
if !interpolations.is_empty() {
return Err(LexFailure::UnterminatedInterpolation);
}
let tail = &src[tiled..];
if !tail.chars().all(char::is_whitespace) {
return Err(LexFailure::NonWhitespaceTail {
tail: tail.to_string(),
});
}
Ok(out)
}