mod error;
mod token;
pub use error::*;
pub use token::*;
use chumsky::input::{Input, Stream};
use logos::Logos;
use crate::parsing::Span;
pub fn lexer(
string: &str,
) -> Result<
(Vec<Span>, Vec<Token>, TokensData),
((Vec<Span>, Vec<Token>, TokensData), Vec<LexingError>),
> {
let mut spans = Vec::new();
let mut tokens = Vec::new();
let mut errors = Vec::new();
let mut lexer = Token::lexer(string);
while let Some(result) = lexer.next() {
let span = Span::new(lexer.span().start as u32, lexer.span().end as u32);
match result {
Ok(token) => {
spans.push(span);
tokens.push(token);
}
Err(err_kind) => errors.push(LexingError {
span: span,
kind: err_kind,
}),
}
}
let eof_pos = string.len() as u32;
spans.push(Span::new(eof_pos, eof_pos));
tokens.push(Token::Eof);
let strings = lexer.extras.strings;
let ints128 = lexer.extras.ints128;
let strings = values_by_id(strings, STRING_ID_OFFSET);
let ints128 = values_by_id(ints128, 0);
let data = TokensData {
ints128: ints128,
strings: strings,
};
match errors.is_empty() {
true => Ok((spans, tokens, data)),
false => Err(((spans, tokens, data), errors)),
}
}
fn values_by_id<T>(values: std::collections::HashMap<T, DataId>, offset: usize) -> Vec<T>
where
T: Eq + std::hash::Hash,
{
let mut ordered = Vec::new();
ordered.resize_with(values.len(), || None);
for (value, id) in values {
let index =
id.0.checked_sub(offset)
.expect("token data id is below its namespace offset");
let slot = ordered
.get_mut(index)
.expect("token data ids must be dense and in bounds");
assert!(slot.replace(value).is_none(), "duplicate token data id");
}
ordered
.into_iter()
.map(|value| value.expect("token data ids must not contain gaps"))
.collect()
}
pub fn make_chumsky_input<'src>(
spans: &'src [Span],
tokens: &'src [Token],
) -> impl Input<'src, Token = Token, Span = Span> {
let eoi_span = spans.last().map_or(Span::new(0, 0), |span| *span);
let iter = tokens.iter().zip(spans.iter());
Stream::from_iter(iter).map(eoi_span, |(token, span)| (*token, *span))
}
#[cfg(test)]
mod tests {
use super::*;
fn tokens(source: &str) -> Vec<Token> {
let (_, mut tokens, _) = lexer(source)
.unwrap_or_else(|(_, errors)| panic!("unexpected lexing errors: {errors:?}"));
assert_eq!(tokens.pop(), Some(Token::Eof));
tokens
}
#[test]
fn lexes_short_suffixed_numbers() {
assert_eq!(
tokens("1u8 10u16 1u32 1u64 1usize 1i8 10i16 1i32 1i64 1isize"),
vec![
Token::Uint8(1),
Token::Uint16(10),
Token::Uint32(1),
Token::Uint64(1),
Token::Usize(1),
Token::Int8(1),
Token::Int16(10),
Token::Int32(1),
Token::Int64(1),
Token::Isize(1),
]
);
}
#[test]
fn lexes_floats_and_separates_unary_minus() {
assert_eq!(
tokens("a-1 1.5 2.25f32 3.5f64"),
vec![
Token::Ident(DataId(STRING_ID_OFFSET)),
Token::Minus,
Token::IntAny(DataId(0)),
Token::FloatAny(1.5),
Token::Float32(2.25),
Token::Float64(3.5),
]
);
}
#[test]
fn lexes_raw_strings_without_truncating_the_payload() {
let (_, mut tokens, data) = lexer(r##"r"plain" r#"hash-✓"#"##)
.unwrap_or_else(|(_, errors)| panic!("unexpected lexing errors: {errors:?}"));
assert_eq!(tokens.pop(), Some(Token::Eof));
assert_eq!(
tokens,
vec![
Token::String(DataId(STRING_ID_OFFSET)),
Token::String(DataId(STRING_ID_OFFSET + 1)),
]
);
assert_eq!(data.strings, ["plain", "hash-✓"]);
}
#[test]
fn skips_line_and_block_comments() {
assert_eq!(
tokens("let /* block\n comment */ value // line comment\n = 1;"),
vec![
Token::LetKw,
Token::Ident(DataId(STRING_ID_OFFSET)),
Token::Eq,
Token::IntAny(DataId(0)),
Token::Semicolon,
]
);
}
#[test]
fn reports_numeric_overflow() {
let (_, errors) = lexer("256u8").expect_err("u8 overflow must be rejected");
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].kind, LexingErrorKind::InvalidUint8Value);
}
#[test]
fn lexes_all_built_in_library_sources() {
let library = crate::core_lib::generate_core_lib();
assert_eq!(library.scripts.len(), 10);
for (path, script) in library.scripts {
lexer(&script.content).unwrap_or_else(|(_, errors)| {
panic!("built-in library `{path}` has lexing errors: {errors:?}")
});
}
}
}