kcl_syntax/
lexer.rs

1//! Logos-based lexer for KCL source.
2
3use std::ops::Range;
4
5use logos::Logos;
6
7use crate::keywords::keyword_kind;
8use crate::syntax_kind::SyntaxKind;
9
10/// Lossless tokenization of a KCL source string.
11///
12/// `LexedSource` keeps the original source and the full token sequence,
13/// including whitespace, comments, and recovery tokens. Lexical errors are
14/// represented as token kinds such as [`SyntaxKind::Unknown`],
15/// [`SyntaxKind::UnterminatedString`], and
16/// [`SyntaxKind::UnterminatedBlockComment`].
17#[derive(Clone, Debug, Eq, PartialEq)]
18pub struct LexedSource<'a> {
19    source: &'a str,
20    tokens: Vec<Token<'a>>,
21}
22
23impl<'a> LexedSource<'a> {
24    /// Returns the original source string.
25    pub fn as_str(&self) -> &'a str {
26        self.source
27    }
28
29    /// Returns all tokens, including trivia and recovery tokens.
30    pub fn tokens(&self) -> &[Token<'a>] {
31        &self.tokens
32    }
33
34    /// Iterates over all tokens by reference.
35    pub fn iter(&self) -> impl Iterator<Item = &Token<'a>> {
36        self.tokens.iter()
37    }
38
39    /// Consumes the lexed source and returns its token vector.
40    pub fn into_tokens(self) -> Vec<Token<'a>> {
41        self.tokens
42    }
43
44    /// Returns the number of tokens.
45    pub fn len(&self) -> usize {
46        self.tokens.len()
47    }
48
49    /// Returns true when the source produced no tokens.
50    pub fn is_empty(&self) -> bool {
51        self.tokens.is_empty()
52    }
53}
54
55impl<'a> IntoIterator for LexedSource<'a> {
56    type IntoIter = std::vec::IntoIter<Token<'a>>;
57    type Item = Token<'a>;
58
59    fn into_iter(self) -> Self::IntoIter {
60        self.tokens.into_iter()
61    }
62}
63
64impl<'a, 's> IntoIterator for &'s LexedSource<'a> {
65    type IntoIter = std::slice::Iter<'s, Token<'a>>;
66    type Item = &'s Token<'a>;
67
68    fn into_iter(self) -> Self::IntoIter {
69        self.tokens.iter()
70    }
71}
72
73/// A single KCL token.
74///
75/// Token text is borrowed from the original source. Ranges are byte offsets into
76/// that source string.
77#[derive(Clone, Debug, Eq, PartialEq)]
78pub struct Token<'a> {
79    kind: SyntaxKind,
80    text: &'a str,
81    range: Range<usize>,
82}
83
84impl<'a> Token<'a> {
85    /// Returns this token's syntax kind.
86    pub fn kind(&self) -> SyntaxKind {
87        self.kind
88    }
89
90    /// Returns the exact source text for this token.
91    pub fn text(&self) -> &'a str {
92        self.text
93    }
94
95    /// Returns this token's byte range in the original source string.
96    pub fn range(&self) -> Range<usize> {
97        self.range.clone()
98    }
99}
100
101#[derive(Clone, Copy, Debug, Logos, PartialEq)]
102enum RawTokenKind {
103    #[regex(r"[ \t\n\r]+")]
104    Whitespace,
105    // - Closed strings may span line boundaries.
106    // - Unterminated strings recover at the next line boundary.
107    #[regex(r#""([^"\\]|\\[\s\S])*""#)]
108    #[regex(r#"'([^'\\]|\\[\s\S])*'"#)]
109    String,
110    #[regex(r#""([^"\\\n\r]|\\[^\n\r])*"#, unterminated_string)]
111    #[regex(r#"'([^'\\\n\r]|\\[^\n\r])*"#, unterminated_string)]
112    UnterminatedString,
113    #[regex(r"//[^\n\r]*", allow_greedy = true)]
114    LineComment,
115    #[regex(r"/\*", block_comment)]
116    BlockComment,
117    #[regex(r"[0-9]+(\.[0-9]+)?_?(mm|cm|m|inch|in|ft|yd|deg|rad|\?)?")]
118    #[regex(r"\.[0-9]+_?(mm|cm|m|inch|in|ft|yd|deg|rad|\?)?")]
119    Number,
120    #[token("..<")]
121    DoublePeriodLessThan,
122    #[token("..")]
123    DoublePeriod,
124    #[token("::")]
125    DoubleColon,
126    #[regex(r"[\p{Alphabetic}_][\p{Alphabetic}0-9_]*")]
127    Word,
128    #[token(">=")]
129    GtEq,
130    #[token("<=")]
131    LtEq,
132    #[token("==")]
133    EqEq,
134    #[token("=>")]
135    FatArrow,
136    #[token("!=")]
137    BangEq,
138    #[token("|>")]
139    PipeGt,
140    #[token("*")]
141    Star,
142    #[token("+")]
143    Plus,
144    #[token("-")]
145    Minus,
146    #[token("/")]
147    Slash,
148    #[token("%")]
149    Percent,
150    #[token("=")]
151    Eq,
152    #[token("<")]
153    Lt,
154    #[token(">")]
155    Gt,
156    #[token("\\")]
157    Backslash,
158    #[token("^")]
159    Caret,
160    #[token("||")]
161    PipePipe,
162    #[token("&&")]
163    AmpAmp,
164    #[token("|")]
165    Pipe,
166    #[token("&")]
167    Amp,
168    #[token("(")]
169    OpenParen,
170    #[token(")")]
171    CloseParen,
172    #[token("{")]
173    OpenBrace,
174    #[token("}")]
175    CloseBrace,
176    #[token("[")]
177    OpenBracket,
178    #[token("]")]
179    CloseBracket,
180    #[token("#")]
181    Hash,
182    #[token("!")]
183    Bang,
184    #[token("$")]
185    Dollar,
186    #[token(",")]
187    Comma,
188    #[token(":")]
189    Colon,
190    #[token(".")]
191    Period,
192    #[token("?")]
193    QuestionMark,
194    #[token("@")]
195    At,
196    #[token(";")]
197    SemiColon,
198}
199
200/// Lexes KCL source into a lossless token sequence.
201///
202/// This function does not return lexical errors separately. Invalid or
203/// recoverable input is preserved in the token stream with recovery token kinds.
204pub fn lex(source: &str) -> LexedSource<'_> {
205    let mut lexer = RawTokenKind::lexer(source);
206    let mut tokens = Vec::new();
207
208    while let Some(raw_kind) = lexer.next() {
209        let range = lexer.span();
210        let text = &source[range.clone()];
211        let kind = match raw_kind {
212            Ok(RawTokenKind::Whitespace) => SyntaxKind::Whitespace,
213            Ok(RawTokenKind::String) => SyntaxKind::String,
214            Ok(RawTokenKind::UnterminatedString) => SyntaxKind::UnterminatedString,
215            Ok(RawTokenKind::LineComment) => SyntaxKind::LineComment,
216            Ok(RawTokenKind::BlockComment) if text.ends_with("*/") => SyntaxKind::BlockComment,
217            Ok(RawTokenKind::BlockComment) => SyntaxKind::UnterminatedBlockComment,
218            Ok(RawTokenKind::Number) => SyntaxKind::Number,
219            Ok(RawTokenKind::DoublePeriodLessThan) => SyntaxKind::DoublePeriodLessThan,
220            Ok(RawTokenKind::DoublePeriod) => SyntaxKind::DoublePeriod,
221            Ok(RawTokenKind::DoubleColon) => SyntaxKind::DoubleColon,
222            Ok(RawTokenKind::Word) => keyword_kind(text).unwrap_or(SyntaxKind::Word),
223            Ok(RawTokenKind::GtEq) => SyntaxKind::GtEq,
224            Ok(RawTokenKind::LtEq) => SyntaxKind::LtEq,
225            Ok(RawTokenKind::EqEq) => SyntaxKind::EqEq,
226            Ok(RawTokenKind::FatArrow) => SyntaxKind::FatArrow,
227            Ok(RawTokenKind::BangEq) => SyntaxKind::BangEq,
228            Ok(RawTokenKind::PipeGt) => SyntaxKind::PipeGt,
229            Ok(RawTokenKind::Star) => SyntaxKind::Star,
230            Ok(RawTokenKind::Plus) => SyntaxKind::Plus,
231            Ok(RawTokenKind::Minus) => SyntaxKind::Minus,
232            Ok(RawTokenKind::Slash) => SyntaxKind::Slash,
233            Ok(RawTokenKind::Percent) => SyntaxKind::Percent,
234            Ok(RawTokenKind::Eq) => SyntaxKind::Eq,
235            Ok(RawTokenKind::Lt) => SyntaxKind::Lt,
236            Ok(RawTokenKind::Gt) => SyntaxKind::Gt,
237            Ok(RawTokenKind::Backslash) => SyntaxKind::Backslash,
238            Ok(RawTokenKind::Caret) => SyntaxKind::Caret,
239            Ok(RawTokenKind::PipePipe) => SyntaxKind::PipePipe,
240            Ok(RawTokenKind::AmpAmp) => SyntaxKind::AmpAmp,
241            Ok(RawTokenKind::Pipe) => SyntaxKind::Pipe,
242            Ok(RawTokenKind::Amp) => SyntaxKind::Amp,
243            Ok(RawTokenKind::OpenParen) => SyntaxKind::OpenParen,
244            Ok(RawTokenKind::CloseParen) => SyntaxKind::CloseParen,
245            Ok(RawTokenKind::OpenBrace) => SyntaxKind::OpenBrace,
246            Ok(RawTokenKind::CloseBrace) => SyntaxKind::CloseBrace,
247            Ok(RawTokenKind::OpenBracket) => SyntaxKind::OpenBracket,
248            Ok(RawTokenKind::CloseBracket) => SyntaxKind::CloseBracket,
249            Ok(RawTokenKind::Hash) => SyntaxKind::Hash,
250            Ok(RawTokenKind::Bang) => SyntaxKind::Bang,
251            Ok(RawTokenKind::Dollar) => SyntaxKind::Dollar,
252            Ok(RawTokenKind::Comma) => SyntaxKind::Comma,
253            Ok(RawTokenKind::Colon) => SyntaxKind::Colon,
254            Ok(RawTokenKind::Period) => SyntaxKind::Period,
255            Ok(RawTokenKind::QuestionMark) => SyntaxKind::QuestionMark,
256            Ok(RawTokenKind::At) => SyntaxKind::At,
257            Ok(RawTokenKind::SemiColon) => SyntaxKind::SemiColon,
258            Err(()) => SyntaxKind::Unknown,
259        };
260        tokens.push(Token { kind, text, range });
261    }
262
263    LexedSource { source, tokens }
264}
265
266fn block_comment(lexer: &mut logos::Lexer<'_, RawTokenKind>) {
267    if let Some(end) = lexer.remainder().find("*/") {
268        lexer.bump(end + 2);
269    } else {
270        lexer.bump(lexer.remainder().len());
271    }
272}
273
274fn unterminated_string(lexer: &mut logos::Lexer<'_, RawTokenKind>) {
275    let until_line_end = lexer
276        .remainder()
277        .find(['\n', '\r'])
278        .unwrap_or_else(|| lexer.remainder().len());
279    lexer.bump(until_line_end);
280}
kcl_syntax/lexer.rs

kcl_syntax/
lexer.rs
