parse_it/
lexer.rs

1//! Lexing for the parser.
2
3use std::cell::Cell;
4use std::hash::Hash;
5
6/// A lexer for the parser.
7pub trait Lexer<'a> {
8    /// The lexed token type.
9    type Token: Copy + Eq;
10    /// The position type.
11    type Position: Copy + Eq + Ord + Hash;
12
13    /// Create a new lexer from the given input.
14    fn new(input: &'a str) -> Self;
15
16    /// Get the current parsing position.
17    fn pos(&self) -> Self::Position;
18
19    /// Consume the next token.
20    fn next(&self) -> (Option<Self::Token>, usize);
21
22    /// Whether the lexer is at the end of the input.
23    fn is_empty(&self) -> bool;
24
25    /// Advance the lexer to the given position.
26    fn advance_to_pos(&self, pos: Self::Position);
27
28    /// Fork the lexer.
29    fn fork(&self) -> Self;
30}
31
32/// A lexer for a single character.
33pub struct CharLexer<'a> {
34    pos: Cell<usize>,
35    remaining: Cell<&'a str>,
36}
37
38impl<'a> Lexer<'a> for CharLexer<'a> {
39    type Token = char;
40    type Position = usize;
41
42    fn new(input: &'a str) -> Self {
43        Self {
44            pos: Cell::new(0),
45            remaining: Cell::new(input),
46        }
47    }
48
49    fn pos(&self) -> Self::Position {
50        self.pos.get()
51    }
52
53    fn next(&self) -> (Option<Self::Token>, usize) {
54        let start = self.pos.get();
55        let mut chars = self.remaining.get().chars();
56        if let Some(c) = chars.next() {
57            let advance = c.len_utf8();
58            let remaining = chars.as_str();
59
60            self.pos.set(start + advance);
61            self.remaining.set(remaining);
62
63            (Some(c), advance)
64        } else {
65            (None, 0)
66        }
67    }
68
69    fn is_empty(&self) -> bool {
70        self.remaining.get().is_empty()
71    }
72
73    fn advance_to_pos(&self, pos: Self::Position) {
74        let advance = pos - self.pos.get();
75        self.pos.set(pos);
76        self.remaining.set(&self.remaining.get()[advance..]);
77    }
78
79    fn fork(&self) -> Self {
80        Self {
81            pos: self.pos.clone(),
82            remaining: self.remaining.clone(),
83        }
84    }
85}
parse_it/lexer.rs

parse_it/
lexer.rs
