parse_it/
lexer.rs

1//! Lexing for the parser.
2
3use std::hash::Hash;
4
5/// A span in the source code.
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub struct Span {
8    /// The start of the span, inclusive
9    pub start: usize,
10    /// The end of the span, exclusive
11    pub end: usize,
12}
13
14/// A lexer for the parser.
15pub trait Lexer<'a> {
16    /// The lexed token type.
17    type Token: Eq;
18    /// The lexeme type.
19    type Position: Clone + Hash + Eq + PartialOrd;
20
21    /// Create a new lexer from the given input.
22    fn new(input: &'a str) -> Self;
23
24    /// Get the current lexeme.
25    fn lexeme(&self) -> &str;
26
27    /// Get the current span.
28    fn span(&self) -> Span;
29
30    /// Get the current position.
31    fn pos(&self) -> &Self::Position;
32
33    /// Consume the next token.
34    fn next(&mut self) -> Option<Self::Token>;
35
36    /// Whether the lexer is at the end of the input.
37    fn is_empty(&self) -> bool;
38
39    /// Advance the lexer to the given lexeme.
40    fn advance_to_pos(&mut self, pos: &Self::Position);
41
42    /// Fork the lexer.
43    fn fork(&self) -> Self;
44}
45
46/// A lexer for a single character.
47#[derive(Clone)]
48pub struct CharLexer<'a> {
49    pos: usize,
50    current: char,
51    remaining: &'a str,
52}
53
54impl<'a> Lexer<'a> for CharLexer<'a> {
55    type Token = char;
56    type Position = usize;
57
58    fn new(input: &'a str) -> Self {
59        Self {
60            pos: 0,
61            current: input.chars().next().unwrap_or_default(),
62            remaining: input,
63        }
64    }
65
66    fn lexeme(&self) -> &str {
67        self.remaining
68    }
69
70    fn span(&self) -> Span {
71        Span {
72            start: self.pos,
73            end: self.pos + self.current.len_utf8(),
74        }
75    }
76
77    fn pos(&self) -> &Self::Position {
78        &self.pos
79    }
80
81    fn next(&mut self) -> Option<Self::Token> {
82        let start = self.pos;
83        let mut chars = self.remaining.chars();
84        if let Some(c) = chars.next() {
85            let advance = c.len_utf8();
86            let remaining = chars.as_str();
87
88            self.pos = start + advance;
89            self.current = c;
90            self.remaining = remaining;
91
92            Some(c)
93        } else {
94            None
95        }
96    }
97
98    fn is_empty(&self) -> bool {
99        self.remaining.is_empty()
100    }
101
102    fn advance_to_pos(&mut self, pos: &usize) {
103        self.pos = *pos;
104        self.remaining = &self.remaining[self.pos..];
105        self.current = self.remaining.chars().next().unwrap_or_default();
106    }
107
108    fn fork(&self) -> Self {
109        self.clone()
110    }
111}
parse_it/lexer.rs

parse_it/
lexer.rs
