reqlang_expr/
lexer.rs

1//! The lexer and associated types
2
3use logos::Logos;
4use std::ops::Range;
5
6use crate::{
7    errors::{ExprErrorS, LexicalError},
8    span::Spanned,
9};
10
11/// Parse source code in to a list of [`Token`].
12pub fn lex(source: &str) -> Vec<Result<(usize, Token, usize), ExprErrorS>> {
13    let lexer: Lexer<'_> = Lexer::new(source);
14    let tokens: Vec<Result<(usize, Token, usize), ExprErrorS>> = lexer.collect::<Vec<_>>();
15
16    tokens
17}
18
19/// Converts a [`String`] source in to a vector of [`Token`]
20#[derive(Debug)]
21pub struct Lexer<'a> {
22    inner: logos::Lexer<'a, Token>,
23    pending: Option<(usize, Token, usize)>,
24}
25
26impl<'a> Lexer<'a> {
27    pub fn new(source: &'a str) -> Self {
28        Self {
29            inner: Token::lexer(source),
30            pending: None,
31        }
32    }
33}
34
35impl Iterator for Lexer<'_> {
36    type Item = Result<(usize, Token, usize), ExprErrorS>;
37
38    fn next(&mut self) -> Option<Self::Item> {
39        if let Some(token) = self.pending.take() {
40            return Some(Ok(token));
41        }
42
43        let token = self.inner.next()?;
44
45        {
46            let Range { start, end } = self.inner.span();
47
48            Some(
49                token
50                    .map(|token| (start, token, end))
51                    .map_err(|(err, err_span)| (err.into(), err_span)),
52            )
53        }
54    }
55}
56
57#[derive(Logos, Debug, Clone, PartialEq)]
58#[logos(error = Spanned<LexicalError>)]
59#[logos(skip r"[ \t\n\f]+")]
60pub enum Token {
61    #[token("(")]
62    LParan,
63
64    #[token(")")]
65    RParan,
66
67    #[token(",")]
68    Comma,
69
70    #[token("<")]
71    LAngle,
72
73    #[token(">")]
74    RAngle,
75
76    #[token("->")]
77    Arrow,
78
79    #[token("Fn")]
80    Fn,
81
82    #[token("...")]
83    ThreeDot,
84
85    #[regex(r#"`[^`]*`"#, lex_string)]
86    String(String),
87
88    #[regex(r#"[0-9]+(\.[0-9]+)?"#, lex_number)]
89    Number(f64),
90
91    #[regex("[!?:@]?[a-z_][a-zA-Z0-9_]*", lex_identifier)]
92    Identifier(String),
93
94    #[regex("[A-Z][a-zA-Z0-9]*", lex_identifier)]
95    Type(String),
96
97    #[token("true")]
98    True,
99
100    #[token("false")]
101    False,
102}
103
104fn lex_identifier(lexer: &mut logos::Lexer<Token>) -> String {
105    let slice = lexer.slice();
106    slice.to_string()
107}
108
109fn lex_string(lexer: &mut logos::Lexer<Token>) -> String {
110    let slice = lexer.slice();
111    slice[1..slice.len() - 1].to_string()
112}
113
114fn lex_number(lexer: &mut logos::Lexer<Token>) -> f64 {
115    let slice = lexer.slice();
116
117    slice.parse::<f64>().expect("should parse as a number")
118}
119
120impl Token {
121    pub fn identifier(identifier: &str) -> Self {
122        Token::Identifier(identifier.to_string())
123    }
124}
reqlang_expr/lexer.rs

reqlang_expr/
lexer.rs
