linkrs_query/parser/lexing/
lexer.rs

1//! Lexer implementation for the query parser
2//!
3//! This module implements a lexical analyzer that converts input query strings into tokens.
4//!
5//! ## Module Structure
6//!
7//! - `scanner` - character-level scanning (identifiers, numbers, strings, comments)
8//! - `token` - token generation and keyword lookup
9
10use crate::parser::lexing::LexError;
11use crate::parser::{Token, TokenKind as Tk};
12use linkrs_core::types::Position;
13use std::iter::Peekable;
14
15mod scanner;
16mod token;
17
18#[derive(Clone)]
19pub struct Lexer<'a> {
20    input: std::borrow::Cow<'a, str>,
21    chars: Peekable<std::vec::IntoIter<char>>,
22    position: usize,
23    line: usize,
24    column: usize,
25    current_token: Token,
26    errors: Vec<LexError>,
27}
28
29pub struct LexerCheckpoint {
30    position: usize,
31    line: usize,
32    column: usize,
33    current_token: Token,
34    chars: Vec<char>,
35}
36
37impl<'a> Lexer<'a> {
38    pub fn new(input: &'a str) -> Self {
39        let chars: Vec<char> = input.chars().collect();
40        let mut lexer = Lexer {
41            input: std::borrow::Cow::Borrowed(input),
42            chars: chars.into_iter().peekable(),
43            position: 0,
44            line: 1,
45            column: 0,
46            current_token: Token::new(Tk::Eof, String::new(), 0, 0),
47            errors: Vec::new(),
48        };
49        lexer.current_token = lexer.next_token();
50        lexer
51    }
52
53    pub fn from_string(input: String) -> Self {
54        let chars: Vec<char> = input.chars().collect();
55        let mut lexer = Lexer {
56            input: std::borrow::Cow::Owned(input),
57            chars: chars.into_iter().peekable(),
58            position: 0,
59            line: 1,
60            column: 0,
61            current_token: Token::new(Tk::Eof, String::new(), 0, 0),
62            errors: Vec::new(),
63        };
64        lexer.read_char();
65        lexer.current_token = lexer.next_token();
66        lexer
67    }
68
69    fn read_char(&mut self) -> Option<char> {
70        let ch = self.chars.next();
71        if let Some(c) = ch {
72            self.position += c.len_utf8();
73            if c == '\n' {
74                self.line += 1;
75                self.column = 0;
76            } else {
77                self.column += 1;
78            }
79        }
80        ch
81    }
82
83    fn peek_char(&mut self) -> Option<&char> {
84        self.chars.peek()
85    }
86
87    fn add_error(&mut self, error: LexError) {
88        self.errors.push(error);
89    }
90
91    pub fn has_errors(&self) -> bool {
92        !self.errors.is_empty()
93    }
94
95    pub fn take_errors(&mut self) -> Vec<LexError> {
96        std::mem::take(&mut self.errors)
97    }
98
99    pub fn errors(&self) -> &[LexError] {
100        &self.errors
101    }
102
103    pub fn checkpoint(&self) -> LexerCheckpoint {
104        LexerCheckpoint {
105            position: self.position,
106            line: self.line,
107            column: self.column,
108            current_token: self.current_token.clone(),
109            chars: self.chars.clone().collect(),
110        }
111    }
112
113    pub fn restore(&mut self, ckpt: LexerCheckpoint) {
114        self.position = ckpt.position;
115        self.line = ckpt.line;
116        self.column = ckpt.column;
117        self.current_token = ckpt.current_token;
118        self.chars = ckpt.chars.into_iter().peekable();
119    }
120
121    pub fn current_token(&self) -> &Token {
122        &self.current_token
123    }
124
125    pub fn current_position(&self) -> Position {
126        Position::new(self.line, self.column)
127    }
128
129    pub fn is_at_end(&mut self) -> bool {
130        self.chars.peek().is_none()
131    }
132
133    pub fn peek(&mut self) -> Result<Token, String> {
134        Ok(self.current_token.clone())
135    }
136
137    pub fn advance(&mut self) {
138        self.current_token = self.next_token();
139    }
140
141    pub fn check(&mut self, kind: Tk) -> bool {
142        self.current_token.kind == kind
143    }
144}
145
146#[cfg(test)]
147mod tests;
linkrs_query/parser/lexing/lexer.rs

linkrs_query/parser/lexing/
lexer.rs
