lehit 0.2.0

Embeddable Lehit language frontend and Cranelift JIT runtime
Documentation
use std::collections::HashMap;

use logos::Logos;

use crate::parsing::lexer::LexingErrorKind;

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DataId(pub usize);

#[derive(Debug, Clone)]
pub struct TokensData {
    pub ints128: Vec<i128>,
    pub strings: Vec<String>,
}

pub const STRING_ID_OFFSET: usize = usize::MAX / 2;

impl TokensData {
    pub fn get_i128(&self, id: DataId) -> i128 {
        self.ints128
            .get(id.0)
            .expect("invalid int128 DataId")
            .to_owned()
    }

    pub fn get_string(&self, id: DataId) -> &str {
        id.0.checked_sub(STRING_ID_OFFSET)
            .and_then(|idx| self.strings.get(idx))
            .expect("invalid string DataId")
    }
}

#[derive(Debug)]
pub struct TokensExtras {
    pub ints128: HashMap<i128, DataId>,
    pub strings: HashMap<String, DataId>,
    _next_ints128_data_id: DataId,
    _next_string_data_id: DataId,
}

impl TokensExtras {
    fn string_id(&mut self, string: &str) -> DataId {
        self.strings.get(string).copied().unwrap_or_else(|| {
            let id = self._next_string_data_id;
            self._next_string_data_id.0 += 1;
            self.strings.insert(string.to_string(), id);
            id
        })
    }

    fn int128_id(&mut self, value: i128) -> DataId {
        self.ints128.get(&value).copied().unwrap_or_else(|| {
            let id = self._next_ints128_data_id;
            self._next_ints128_data_id.0 += 1;
            self.ints128.insert(value, id);
            id
        })
    }
}

impl Default for TokensExtras {
    fn default() -> Self {
        Self {
            ints128: HashMap::new(),
            strings: HashMap::new(),
            _next_ints128_data_id: DataId(0),
            _next_string_data_id: DataId(usize::MAX / 2),
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Logos)]
#[logos(error = LexingErrorKind)]
#[logos(skip r"[ \t\r\n\f]+")]
#[logos(skip(r"//[^\n\r]*", allow_greedy = true))]
#[logos(skip r"/\*([^*]|\*[^/])*\*/")]
#[logos(extras = TokensExtras)]
pub enum Token {
    // keywords
    #[token("unsafe")]
    UnsafeKw,
    #[token("pub")]
    PubKw,
    #[token("fn")]
    FnKw,
    #[token("type")]
    TypeKw,
    #[token("struct")]
    StructKw,
    #[token("enum")]
    EnumKw,
    #[token("trait")]
    TraitKw,
    #[token("dyn")]
    DynKw,
    #[token("impl")]
    ImplKw,
    #[token("extension")]
    ExtensionKw,
    #[token("patch")]
    PatchKw,
    #[token("as")]
    AsKw,
    #[token("from")]
    FromKw,
    #[token("const")]
    ConstKw,
    #[token("mut")]
    MutKw,
    #[token("acs")]
    AcsKw,
    #[token("fld")]
    FldKw,
    #[token("var")]
    VarKw,
    #[token("let")]
    LetKw,
    #[token("if")]
    IfKw,
    #[token("match")]
    MatchKw,
    #[token("else")]
    ElseKw,
    #[token("loop")]
    LoopKw,
    #[token("while")]
    WhileKw,
    #[token("for")]
    ForKw,
    #[token("in")]
    InKw,
    #[token("return")]
    ReturnKw,
    #[token("self")]
    LowSelfKw,
    #[token("Self")]
    BigSelfKw,
    #[token("true")]
    TrueKw,
    #[token("false")]
    FalseKw,

    // punctuation / operators
    #[token(".")]
    Dot,
    #[token(",")]
    Comma,
    #[token(":")]
    Colon,
    #[token(";")]
    Semicolon,
    #[token("'")]
    SingleQuote,
    #[token("\"")]
    DoubleQuote,
    #[token("->")]
    Arrow,
    #[token("=>")]
    FatArrow,
    #[token("&")]
    Ampersand,
    #[token("=")]
    Eq,
    #[token("==")]
    EqEq,
    #[token("!=")]
    NotEq,
    #[token("!")]
    Bang,
    #[token("+")]
    Plus,
    #[token("+=")]
    PlusEq,
    #[token("-")]
    Minus,
    #[token("-=")]
    MinusEq,
    #[token("*")]
    Asterisk,
    #[token("*=")]
    AsteriskEq,
    #[token("/")]
    Slash,
    #[token("/=")]
    SlashEq,
    #[token("<<=")]
    ShiftLeftEq,
    #[token(">>=")]
    ShiftRightEq,

    #[token("<")]
    LeftAngle,
    #[token("<=")]
    LeftAngleEq,
    #[token(">")]
    RightAngle,
    #[token(">=")]
    RightAngleEq,

    #[token("(")]
    OpenParen,
    #[token(")")]
    CloseParen,
    #[token("[")]
    OpenBracket,
    #[token("]")]
    CloseBracket,
    #[token("{")]
    OpenBrace,
    #[token("}")]
    CloseBrace,

    // Numeric values. A leading minus is deliberately tokenized separately so
    // `a-1` and `a - 1` have identical token streams; unary negation belongs to
    // the expression grammar.
    #[regex(r"[0-9][0-9_]*u8", |lex| parse_number::<u8>(lex.slice(), "u8", LexingErrorKind::InvalidUint8Value))]
    Uint8(u8),
    #[regex(r"[0-9][0-9_]*u16", |lex| parse_number::<u16>(lex.slice(), "u16", LexingErrorKind::InvalidUint16Value))]
    Uint16(u16),
    #[regex(r"[0-9][0-9_]*u32", |lex| parse_number::<u32>(lex.slice(), "u32", LexingErrorKind::InvalidUint32Value))]
    Uint32(u32),
    #[regex(r"[0-9][0-9_]*u64", |lex| parse_number::<u64>(lex.slice(), "u64", LexingErrorKind::InvalidUint64Value))]
    Uint64(u64),
    #[regex(r"[0-9][0-9_]*usize", |lex| parse_number::<usize>(lex.slice(), "usize", LexingErrorKind::InvalidUsizeValue))]
    Usize(usize),
    #[regex(r"[0-9][0-9_]*i8", |lex| parse_number::<i8>(lex.slice(), "i8", LexingErrorKind::InvalidInt8Value))]
    Int8(i8),
    #[regex(r"[0-9][0-9_]*i16", |lex| parse_number::<i16>(lex.slice(), "i16", LexingErrorKind::InvalidInt16Value))]
    Int16(i16),
    #[regex(r"[0-9][0-9_]*i32", |lex| parse_number::<i32>(lex.slice(), "i32", LexingErrorKind::InvalidInt32Value))]
    Int32(i32),
    #[regex(r"[0-9][0-9_]*i64", |lex| parse_number::<i64>(lex.slice(), "i64", LexingErrorKind::InvalidInt64Value))]
    Int64(i64),
    #[regex(r"[0-9][0-9_]*isize", |lex| parse_number::<isize>(lex.slice(), "isize", LexingErrorKind::InvalidIsizeValue))]
    Isize(isize),

    #[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*f32", |lex| parse_number::<f32>(lex.slice(), "f32", LexingErrorKind::InvalidFloat32Value))]
    Float32(f32),
    #[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*f64", |lex| parse_number::<f64>(lex.slice(), "f64", LexingErrorKind::InvalidFloat64Value))]
    Float64(f64),
    #[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*", |lex| parse_number::<f64>(lex.slice(), "", LexingErrorKind::InvalidFloatAnyValue))]
    FloatAny(f64),
    #[regex(r"[0-9][0-9_]*", |lex| parse_number::<i128>(lex.slice(), "", LexingErrorKind::InvalidIntAnyValue).map(|value| lex.extras.int128_id(value)))]
    IntAny(DataId),

    // strings / chars
    #[regex(r#"'([^'\\]|\\.)'"#, |lex| handle_char(lex))]
    Char(char),

    #[regex(r#""([^"\\]|\\.)*""#, |lex| {
        let string = &handle_string(lex);
        lex.extras.string_id(string)
    })]
    #[regex(r#"r"([^"]*)""#, |lex| {
            let slice = lex.slice();
            lex.extras.string_id(&slice[2..slice.len() - 1])
        })]
    #[regex(r##"r#"([^"]*)"#"##, |lex| {
            let slice = lex.slice();
            lex.extras.string_id(&slice[3..slice.len() - 2])
        })]
    String(DataId),

    // ident
    #[regex(r"[A-Za-z_][A-Za-z0-9_]*", |lex| {
        lex.extras.string_id(lex.slice())
    })]
    Ident(DataId),

    #[regex(r"'[A-Za-z_][A-Za-z0-9_]*", |lex| {
        lex.extras.string_id(&lex.slice()[1..])
    })]
    Lifetime(DataId),

    // synthetic tokens
    Eof,
}

fn parse_number<T: std::str::FromStr>(
    slice: &str,
    suffix: &str,
    error: LexingErrorKind,
) -> Result<T, LexingErrorKind> {
    let digits = slice
        .strip_suffix(suffix)
        .expect("numeric lexer rule must match its suffix")
        .replace('_', "");
    digits.parse().map_err(|_| error)
}

fn handle_char(lex: &mut logos::Lexer<Token>) -> char {
    let slice = lex.slice();
    let inner = &slice[1..slice.len() - 1];

    if let Some(escaped) = inner.strip_prefix('\\') {
        match escaped {
            "n" => '\n',
            "t" => '\t',
            "r" => '\r',
            "\\" => '\\',
            "'" => '\'',
            "\"" => '"',
            other => other.chars().next().unwrap_or_default(),
        }
    } else {
        inner.chars().next().unwrap_or_default()
    }
}

fn handle_string(lex: &mut logos::Lexer<Token>) -> String {
    let slice = lex.slice();
    let inner = &slice[1..slice.len() - 1];

    let mut result = String::with_capacity(inner.len());
    let mut chars = inner.chars().peekable();

    while let Some(c) = chars.next() {
        if c == '\\' {
            match chars.next() {
                Some('n') => result.push('\n'),
                Some('t') => result.push('\t'),
                Some('r') => result.push('\r'),
                Some('\\') => result.push('\\'),
                Some('"') => result.push('"'),
                Some(other) => result.push(other),
                None => {}
            }
        } else {
            result.push(c);
        }
    }
    result
}