cqlib_core/compile/knowledge/rule_dsl/
lexer.rs

1// This code is part of Cqlib.
2//
3// (C) Copyright China Telecom Quantum Group 2025-2026
4//
5// This code is licensed under the Apache License, Version 2.0. You may
6// obtain a copy of this license in the LICENSE.txt file in the root directory
7// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
8//
9// Any modifications or derivative works of this code must retain this
10// copyright notice, and modified files need to carry a notice indicating
11// that they have been altered from the originals.
12
13//! Lexer for the rule DSL.
14//!
15//! The lexer produces tokens that borrow from the original input string,
16//! making it cheap to extract expression spans for delegation to
17//! [`Parameter::try_from`](crate::circuit::Parameter).
18
19/// Classification of a token in the rule DSL.
20#[derive(Debug, Clone, PartialEq)]
21pub enum TokenKind {
22    /// Keyword `rule`.
23    Rule,
24    /// Keyword `match`.
25    Match,
26    /// Keyword `require`.
27    Require,
28    /// Keyword `rewrite`.
29    Rewrite,
30    /// Keyword `mod` (used in `require { a == b mod c }`).
31    Mod,
32    /// An identifier, e.g. `merge_rz`, `a`, `π`.
33    Ident,
34    /// A numeric literal, e.g. `0`, `2.5`, `3.14`.
35    Number,
36    /// Left brace `{`.
37    LBrace,
38    /// Right brace `}`.
39    RBrace,
40    /// Left parenthesis `(`.
41    LParen,
42    /// Right parenthesis `)`.
43    RParen,
44    /// Left bracket `[`.
45    LBracket,
46    /// Right bracket `]`.
47    RBracket,
48    /// Comma `,`.
49    Comma,
50    /// A line break.
51    Newline,
52    /// Equality operator `==`.
53    EqEq,
54    /// Plus operator `+`.
55    Plus,
56    /// Minus operator `-`.
57    Minus,
58    /// Star operator `*`.
59    Star,
60    /// Slash operator `/`.
61    Slash,
62    /// End of input.
63    Eof,
64}
65
66/// A single lexical token.
67///
68/// Tokens are zero-copy: `text` borrows directly from the original input
69/// string, so extracting expression spans is allocation-free.
70#[derive(Debug, Clone, PartialEq)]
71pub struct Token<'a> {
72    /// The kind of token.
73    pub kind: TokenKind,
74    /// The raw text slice from the input string.
75    pub text: &'a str,
76    /// Byte offset of the start of this token in the input string.
77    pub pos: usize,
78}
79
80/// Error produced by the lexer when an unexpected character is encountered.
81#[derive(Debug, Clone, PartialEq, thiserror::Error)]
82#[error("lexer error at byte {pos}: {msg}")]
83pub struct LexError {
84    /// Human-readable error message.
85    pub msg: String,
86    /// Byte offset in the input where the error occurred.
87    pub pos: usize,
88}
89
90/// Hand-written character scanner for the rule DSL.
91///
92/// # Example
93///
94/// ```rust
95/// use cqlib_core::compile::knowledge::rule_dsl::lexer::{Lexer, TokenKind};
96///
97/// let mut lexer = Lexer::new("rule foo { match {} rewrite {} }");
98/// assert_eq!(lexer.next_token().unwrap().kind, TokenKind::Rule);
99/// assert_eq!(lexer.next_token().unwrap().kind, TokenKind::Ident);
100/// ```
101pub struct Lexer<'a> {
102    /// The full input string being tokenized.
103    input: &'a str,
104    /// Current byte offset into `input`.
105    pos: usize,
106}
107
108impl<'a> Lexer<'a> {
109    /// Creates a new lexer positioned at the start of `input`.
110    pub fn new(input: &'a str) -> Self {
111        Self { input, pos: 0 }
112    }
113
114    /// Returns the next token from the input.
115    ///
116    /// Horizontal whitespace and `//` line comments are skipped automatically.
117    /// Line breaks are emitted as [`TokenKind::Newline`].
118    /// Returns [`TokenKind::Eof`] when the end of the input is reached.
119    pub fn next_token(&mut self) -> Result<Token<'a>, LexError> {
120        self.skip_horizontal_whitespace_and_comments();
121
122        if self.pos >= self.input.len() {
123            return Ok(Token {
124                kind: TokenKind::Eof,
125                text: "",
126                pos: self.pos,
127            });
128        }
129
130        let start = self.pos;
131        let ch = self.peek_char().unwrap();
132
133        match ch {
134            '\n' | '\r' => {
135                self.advance();
136                if ch == '\r' && self.peek_char() == Some('\n') {
137                    self.advance();
138                }
139                // Collapse consecutive line breaks, including mixed CRLF and LF input.
140                loop {
141                    self.skip_horizontal_whitespace_and_comments();
142                    match self.peek_char() {
143                        Some('\n') => {
144                            self.advance();
145                        }
146                        Some('\r') => {
147                            self.advance();
148                            if self.peek_char() == Some('\n') {
149                                self.advance();
150                            }
151                        }
152                        _ => break,
153                    }
154                }
155                self.make_token(TokenKind::Newline, start)
156            }
157            '{' => {
158                self.advance();
159                self.make_token(TokenKind::LBrace, start)
160            }
161            '}' => {
162                self.advance();
163                self.make_token(TokenKind::RBrace, start)
164            }
165            '(' => {
166                self.advance();
167                self.make_token(TokenKind::LParen, start)
168            }
169            ')' => {
170                self.advance();
171                self.make_token(TokenKind::RParen, start)
172            }
173            '[' => {
174                self.advance();
175                self.make_token(TokenKind::LBracket, start)
176            }
177            ']' => {
178                self.advance();
179                self.make_token(TokenKind::RBracket, start)
180            }
181            ',' => {
182                self.advance();
183                self.make_token(TokenKind::Comma, start)
184            }
185            '+' => {
186                self.advance();
187                self.make_token(TokenKind::Plus, start)
188            }
189            '-' => {
190                self.advance();
191                self.make_token(TokenKind::Minus, start)
192            }
193            '*' => {
194                self.advance();
195                self.make_token(TokenKind::Star, start)
196            }
197            '/' => {
198                self.advance();
199                if self.peek_char() == Some('/') {
200                    self.skip_line_comment();
201                    self.next_token()
202                } else {
203                    self.make_token(TokenKind::Slash, start)
204                }
205            }
206            '=' => {
207                self.advance();
208                if self.peek_char() == Some('=') {
209                    self.advance();
210                    self.make_token(TokenKind::EqEq, start)
211                } else {
212                    Err(LexError {
213                        msg: format!("unexpected character '{}'", ch),
214                        pos: start,
215                    })
216                }
217            }
218            _ if ch.is_alphabetic() || ch == '_' => self.read_ident(start),
219            _ if ch.is_ascii_digit() || ch == '.' => self.read_number(start),
220            _ => Err(LexError {
221                msg: format!("unexpected character '{}'", ch),
222                pos: start,
223            }),
224        }
225    }
226
227    /// Reads an identifier or keyword starting at `start`.
228    fn read_ident(&mut self, start: usize) -> Result<Token<'a>, LexError> {
229        while let Some(ch) = self.peek_char() {
230            if ch.is_alphanumeric() || ch == '_' {
231                self.advance();
232            } else {
233                break;
234            }
235        }
236        let text = &self.input[start..self.pos];
237        let kind = match text {
238            "rule" => TokenKind::Rule,
239            "match" => TokenKind::Match,
240            "require" => TokenKind::Require,
241            "rewrite" => TokenKind::Rewrite,
242            "mod" => TokenKind::Mod,
243            _ => TokenKind::Ident,
244        };
245        Ok(Token {
246            kind,
247            text,
248            pos: start,
249        })
250    }
251
252    /// Reads a numeric literal starting at `start`.
253    fn read_number(&mut self, start: usize) -> Result<Token<'a>, LexError> {
254        while let Some(ch) = self.peek_char() {
255            if ch.is_ascii_digit() || ch == '.' {
256                self.advance();
257            } else {
258                break;
259            }
260        }
261        let text = &self.input[start..self.pos];
262        Ok(Token {
263            kind: TokenKind::Number,
264            text,
265            pos: start,
266        })
267    }
268
269    /// Skips whitespace and line comments until the next meaningful character.
270    fn skip_horizontal_whitespace_and_comments(&mut self) {
271        loop {
272            match self.peek_char() {
273                Some(ch) if ch.is_whitespace() && ch != '\n' && ch != '\r' => self.advance(),
274                Some('/') if self.peek_next() == Some('/') => self.skip_line_comment(),
275                _ => break,
276            }
277        }
278    }
279
280    /// Skips from `//` to the end of the current line.
281    fn skip_line_comment(&mut self) {
282        while let Some(ch) = self.peek_char() {
283            if ch == '\n' {
284                break;
285            }
286            self.advance();
287        }
288    }
289
290    /// Builds a [`Token`] of the given kind spanning from `start` to the
291    /// current cursor position.
292    fn make_token(&self, kind: TokenKind, start: usize) -> Result<Token<'a>, LexError> {
293        Ok(Token {
294            kind,
295            text: &self.input[start..self.pos],
296            pos: start,
297        })
298    }
299
300    /// Returns the next character without consuming it.
301    fn peek_char(&self) -> Option<char> {
302        self.input[self.pos..].chars().next()
303    }
304
305    /// Returns the character after the next one without consuming anything.
306    fn peek_next(&self) -> Option<char> {
307        self.input[self.pos..].chars().nth(1)
308    }
309
310    /// Consumes one UTF-8 character and advances the cursor.
311    fn advance(&mut self) {
312        if let Some(ch) = self.peek_char() {
313            self.pos += ch.len_utf8();
314        }
315    }
316}
317
318#[cfg(test)]
319mod tests {
320    use super::*;
321
322    fn lex_all(input: &str) -> Vec<TokenKind> {
323        let mut lexer = Lexer::new(input);
324        let mut kinds = Vec::new();
325        loop {
326            let token = lexer.next_token().unwrap();
327            kinds.push(token.kind.clone());
328            if token.kind == TokenKind::Eof {
329                break;
330            }
331        }
332        kinds
333    }
334
335    #[test]
336    fn lex_keywords_and_ident() {
337        let kinds = lex_all("rule match require rewrite mod foo");
338        assert_eq!(
339            kinds,
340            vec![
341                TokenKind::Rule,
342                TokenKind::Match,
343                TokenKind::Require,
344                TokenKind::Rewrite,
345                TokenKind::Mod,
346                TokenKind::Ident,
347                TokenKind::Eof,
348            ]
349        );
350    }
351
352    #[test]
353    fn lex_punctuation() {
354        let kinds = lex_all("{ } ( ) [ ] , \n == + - * /");
355        assert_eq!(
356            kinds,
357            vec![
358                TokenKind::LBrace,
359                TokenKind::RBrace,
360                TokenKind::LParen,
361                TokenKind::RParen,
362                TokenKind::LBracket,
363                TokenKind::RBracket,
364                TokenKind::Comma,
365                TokenKind::Newline,
366                TokenKind::EqEq,
367                TokenKind::Plus,
368                TokenKind::Minus,
369                TokenKind::Star,
370                TokenKind::Slash,
371                TokenKind::Eof,
372            ]
373        );
374    }
375
376    #[test]
377    fn lex_numbers() {
378        let kinds = lex_all("0 1 2.5 3.14");
379        assert_eq!(
380            kinds,
381            vec![
382                TokenKind::Number,
383                TokenKind::Number,
384                TokenKind::Number,
385                TokenKind::Number,
386                TokenKind::Eof,
387            ]
388        );
389    }
390
391    #[test]
392    fn lex_comment() {
393        let kinds = lex_all("rule // this is a comment\nmatch");
394        assert_eq!(
395            kinds,
396            vec![
397                TokenKind::Rule,
398                TokenKind::Newline,
399                TokenKind::Match,
400                TokenKind::Eof
401            ]
402        );
403    }
404
405    #[test]
406    fn lex_consecutive_newlines() {
407        let kinds = lex_all("rule\n\n\r\nmatch");
408        assert_eq!(
409            kinds,
410            vec![
411                TokenKind::Rule,
412                TokenKind::Newline,
413                TokenKind::Match,
414                TokenKind::Eof
415            ]
416        );
417    }
418
419    #[test]
420    fn lex_unicode_identifier_accepted() {
421        // Greek and CJK characters are alphabetic per Unicode, so they are identifiers.
422        let kinds = lex_all("rule α");
423        assert_eq!(
424            kinds,
425            vec![
426                TokenKind::Rule,
427                TokenKind::Ident, // α (U+03B1)
428                TokenKind::Eof
429            ]
430        );
431    }
432}
cqlib_core/compile/knowledge/rule_dsl/lexer.rs

cqlib_core/compile/knowledge/rule_dsl/
lexer.rs
