reqlang_expr/
lexer.rs

1//! The lexer and associated types
2
3use logos::Logos;
4use std::ops::Range;
5
6use crate::errors::{self, ExprError};
7
8/// Converts a [`String`] source in to a vector of [`Token`]
9#[derive(Debug)]
10pub struct Lexer<'a> {
11    inner: logos::Lexer<'a, Token>,
12    pending: Option<(usize, Token, usize)>,
13}
14
15impl<'a> Lexer<'a> {
16    pub fn new(source: &'a str) -> Self {
17        Self {
18            inner: Token::lexer(source),
19            pending: None,
20        }
21    }
22}
23
24impl Iterator for Lexer<'_> {
25    type Item = Result<(usize, Token, usize), (ExprError, Range<usize>)>;
26
27    fn next(&mut self) -> Option<Self::Item> {
28        if let Some(token) = self.pending.take() {
29            return Some(Ok(token));
30        }
31
32        let token = self.inner.next()?;
33
34        {
35            let Range { start, end } = self.inner.span();
36
37            Some(
38                token
39                    .map(|token| (start, token, end))
40                    .map_err(|(err, err_span)| (err.into(), err_span)),
41            )
42        }
43    }
44}
45
46#[derive(Logos, Debug, Clone, PartialEq)]
47#[logos(error = (errors::LexicalError, Range<usize>))]
48#[logos(skip r"[ \t\n\f]+")]
49pub enum Token {
50    #[token("(")]
51    LParan,
52
53    #[token(")")]
54    RParan,
55
56    #[regex(r#"`[^`]*`"#, lex_string)]
57    String(String),
58
59    #[regex("[!?:@]?[a-zA-Z_][a-zA-Z0-9_]*", lex_identifier)]
60    Identifier(String),
61
62    #[token("true")]
63    True,
64
65    #[token("false")]
66    False,
67}
68
69fn lex_identifier(lexer: &mut logos::Lexer<Token>) -> String {
70    let slice = lexer.slice();
71    slice.to_string()
72}
73
74fn lex_string(lexer: &mut logos::Lexer<Token>) -> String {
75    let slice = lexer.slice();
76    slice[1..slice.len() - 1].to_string()
77}
78
79impl Token {
80    pub fn identifier(identifier: &str) -> Self {
81        Token::Identifier(identifier.to_string())
82    }
83}
reqlang_expr/lexer.rs

reqlang_expr/
lexer.rs
