mest_core/
lexer.rs

1use std::ops::Range;
2
3use crate::token::Token;
4use logos::Logos;
5
6#[derive(Debug)]
7pub struct LexError {
8    pub span: Range<usize>,
9    pub message: Option<String>,
10}
11
12pub fn tokenize(src: &str) -> (Vec<(Token, Range<usize>)>, Vec<LexError>) {
13    let mut tokens = vec![];
14    let mut errors = vec![];
15
16    for (result, span) in Token::lexer(src).spanned() {
17        match result {
18            Ok(tok) => tokens.push((tok, span)),
19            Err(_) => errors.push(LexError {
20                message: Some(format!("unexpected character `{}`", &src[span.clone()])),
21                span,
22            }),
23        }
24    }
25
26    (tokens, errors)
27}
mest_core/lexer.rs

mest_core/
lexer.rs
