lehit 0.2.0

Embeddable Lehit language frontend and Cranelift JIT runtime
Documentation
mod error;
mod token;

pub use error::*;
pub use token::*;

use chumsky::input::{Input, Stream};
use logos::Logos;

use crate::parsing::Span;

pub fn lexer(
    string: &str,
) -> Result<
    (Vec<Span>, Vec<Token>, TokensData),
    ((Vec<Span>, Vec<Token>, TokensData), Vec<LexingError>),
> {
    let mut spans = Vec::new();
    let mut tokens = Vec::new();
    let mut errors = Vec::new();

    let mut lexer = Token::lexer(string);

    while let Some(result) = lexer.next() {
        let span = Span::new(lexer.span().start as u32, lexer.span().end as u32);

        match result {
            Ok(token) => {
                spans.push(span);
                tokens.push(token);
            }
            Err(err_kind) => errors.push(LexingError {
                span: span,
                kind: err_kind,
            }),
        }
    }

    let eof_pos = string.len() as u32;
    spans.push(Span::new(eof_pos, eof_pos));
    tokens.push(Token::Eof);

    let strings = lexer.extras.strings;
    let ints128 = lexer.extras.ints128;

    let strings = values_by_id(strings, STRING_ID_OFFSET);
    let ints128 = values_by_id(ints128, 0);

    let data = TokensData {
        ints128: ints128,
        strings: strings,
    };

    match errors.is_empty() {
        true => Ok((spans, tokens, data)),
        false => Err(((spans, tokens, data), errors)),
    }
}

fn values_by_id<T>(values: std::collections::HashMap<T, DataId>, offset: usize) -> Vec<T>
where
    T: Eq + std::hash::Hash,
{
    let mut ordered = Vec::new();
    ordered.resize_with(values.len(), || None);
    for (value, id) in values {
        let index =
            id.0.checked_sub(offset)
                .expect("token data id is below its namespace offset");
        let slot = ordered
            .get_mut(index)
            .expect("token data ids must be dense and in bounds");
        assert!(slot.replace(value).is_none(), "duplicate token data id");
    }
    ordered
        .into_iter()
        .map(|value| value.expect("token data ids must not contain gaps"))
        .collect()
}

pub fn make_chumsky_input<'src>(
    spans: &'src [Span],
    tokens: &'src [Token],
) -> impl Input<'src, Token = Token, Span = Span> {
    let eoi_span = spans.last().map_or(Span::new(0, 0), |span| *span);

    let iter = tokens.iter().zip(spans.iter());

    Stream::from_iter(iter).map(eoi_span, |(token, span)| (*token, *span))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn tokens(source: &str) -> Vec<Token> {
        let (_, mut tokens, _) = lexer(source)
            .unwrap_or_else(|(_, errors)| panic!("unexpected lexing errors: {errors:?}"));
        assert_eq!(tokens.pop(), Some(Token::Eof));
        tokens
    }

    #[test]
    fn lexes_short_suffixed_numbers() {
        assert_eq!(
            tokens("1u8 10u16 1u32 1u64 1usize 1i8 10i16 1i32 1i64 1isize"),
            vec![
                Token::Uint8(1),
                Token::Uint16(10),
                Token::Uint32(1),
                Token::Uint64(1),
                Token::Usize(1),
                Token::Int8(1),
                Token::Int16(10),
                Token::Int32(1),
                Token::Int64(1),
                Token::Isize(1),
            ]
        );
    }

    #[test]
    fn lexes_floats_and_separates_unary_minus() {
        assert_eq!(
            tokens("a-1 1.5 2.25f32 3.5f64"),
            vec![
                Token::Ident(DataId(STRING_ID_OFFSET)),
                Token::Minus,
                Token::IntAny(DataId(0)),
                Token::FloatAny(1.5),
                Token::Float32(2.25),
                Token::Float64(3.5),
            ]
        );
    }

    #[test]
    fn lexes_raw_strings_without_truncating_the_payload() {
        let (_, mut tokens, data) = lexer(r##"r"plain" r#"hash-✓"#"##)
            .unwrap_or_else(|(_, errors)| panic!("unexpected lexing errors: {errors:?}"));
        assert_eq!(tokens.pop(), Some(Token::Eof));
        assert_eq!(
            tokens,
            vec![
                Token::String(DataId(STRING_ID_OFFSET)),
                Token::String(DataId(STRING_ID_OFFSET + 1)),
            ]
        );
        assert_eq!(data.strings, ["plain", "hash-✓"]);
    }

    #[test]
    fn skips_line_and_block_comments() {
        assert_eq!(
            tokens("let /* block\n comment */ value // line comment\n = 1;"),
            vec![
                Token::LetKw,
                Token::Ident(DataId(STRING_ID_OFFSET)),
                Token::Eq,
                Token::IntAny(DataId(0)),
                Token::Semicolon,
            ]
        );
    }

    #[test]
    fn reports_numeric_overflow() {
        let (_, errors) = lexer("256u8").expect_err("u8 overflow must be rejected");
        assert_eq!(errors.len(), 1);
        assert_eq!(errors[0].kind, LexingErrorKind::InvalidUint8Value);
    }

    #[test]
    fn lexes_all_built_in_library_sources() {
        let library = crate::core_lib::generate_core_lib();
        assert_eq!(library.scripts.len(), 10);

        for (path, script) in library.scripts {
            lexer(&script.content).unwrap_or_else(|(_, errors)| {
                panic!("built-in library `{path}` has lexing errors: {errors:?}")
            });
        }
    }
}