mest-core 0.2.0

Core language library for the Mest programming language
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

use std::ops::Range;

use crate::token::Token;
use logos::Logos;

#[derive(Debug)]
pub struct LexError {
    pub span: Range<usize>,
    pub message: Option<String>,
}

pub fn tokenize(src: &str) -> (Vec<(Token, Range<usize>)>, Vec<LexError>) {
    let mut tokens = vec![];
    let mut errors = vec![];

    for (result, span) in Token::lexer(src).spanned() {
        match result {
            Ok(tok) => tokens.push((tok, span)),
            Err(_) => errors.push(LexError {
                message: Some(format!("unexpected character `{}`", &src[span.clone()])),
                span,
            }),
        }
    }

    (tokens, errors)
}