kcl_syntax/
lexer.rs

1//! Logos-based lexer for KCL source.
2
3use std::ops::Range;
4
5use logos::Logos;
6
7use crate::syntax_kind::SyntaxKind;
8
9/// Lossless tokenization of a KCL source string.
10///
11/// `LexedSource` keeps the original source and the full token sequence,
12/// including whitespace, comments, and recovery tokens. Lexical errors are
13/// represented as token kinds such as [`SyntaxKind::Unknown`],
14/// [`SyntaxKind::UnterminatedString`], and
15/// [`SyntaxKind::UnterminatedBlockComment`].
16#[derive(Clone, Debug, Eq, PartialEq)]
17pub struct LexedSource<'a> {
18    source: &'a str,
19    tokens: Vec<Token<'a>>,
20}
21
22impl<'a> LexedSource<'a> {
23    /// Returns the original source string.
24    pub fn as_str(&self) -> &'a str {
25        self.source
26    }
27
28    /// Returns all tokens, including trivia and recovery tokens.
29    pub fn tokens(&self) -> &[Token<'a>] {
30        &self.tokens
31    }
32
33    /// Iterates over all tokens by reference.
34    pub fn iter(&self) -> impl Iterator<Item = &Token<'a>> {
35        self.tokens.iter()
36    }
37
38    /// Consumes the lexed source and returns its token vector.
39    pub fn into_tokens(self) -> Vec<Token<'a>> {
40        self.tokens
41    }
42
43    /// Returns the number of tokens.
44    pub fn len(&self) -> usize {
45        self.tokens.len()
46    }
47
48    /// Returns true when the source produced no tokens.
49    pub fn is_empty(&self) -> bool {
50        self.tokens.is_empty()
51    }
52}
53
54impl<'a> IntoIterator for LexedSource<'a> {
55    type IntoIter = std::vec::IntoIter<Token<'a>>;
56    type Item = Token<'a>;
57
58    fn into_iter(self) -> Self::IntoIter {
59        self.tokens.into_iter()
60    }
61}
62
63impl<'a, 's> IntoIterator for &'s LexedSource<'a> {
64    type IntoIter = std::slice::Iter<'s, Token<'a>>;
65    type Item = &'s Token<'a>;
66
67    fn into_iter(self) -> Self::IntoIter {
68        self.tokens.iter()
69    }
70}
71
72/// A single KCL token.
73///
74/// Token text is borrowed from the original source. Ranges are byte offsets into
75/// that source string.
76#[derive(Clone, Debug, Eq, PartialEq)]
77pub struct Token<'a> {
78    kind: SyntaxKind,
79    text: &'a str,
80    range: Range<usize>,
81}
82
83impl<'a> Token<'a> {
84    /// Returns this token's syntax kind.
85    pub fn kind(&self) -> SyntaxKind {
86        self.kind
87    }
88
89    /// Returns the exact source text for this token.
90    pub fn text(&self) -> &'a str {
91        self.text
92    }
93
94    /// Returns this token's byte range in the original source string.
95    pub fn range(&self) -> Range<usize> {
96        self.range.clone()
97    }
98}
99
100#[derive(Clone, Copy, Debug, Logos, PartialEq)]
101enum RawTokenKind {
102    #[regex(r"[ \t\n\r]+")]
103    Whitespace,
104    // A closed string may span newlines (KCL supports multiline strings), matching
105    // the legacy tokeniser: content is any char except the quote or backslash, or a
106    // backslash-escape of any char including a newline. Note the UnterminatedString
107    // patterns below deliberately still stop at a line boundary for recovery.
108    #[regex(r#""([^"\\]|\\[\s\S])*""#)]
109    #[regex(r#"'([^'\\]|\\[\s\S])*'"#)]
110    String,
111    #[regex(r#""([^"\\\n\r]|\\[^\n\r])*"#, unterminated_string)]
112    #[regex(r#"'([^'\\\n\r]|\\[^\n\r])*"#, unterminated_string)]
113    UnterminatedString,
114    #[regex(r"//[^\n\r]*", allow_greedy = true)]
115    LineComment,
116    #[regex(r"/\*", block_comment)]
117    BlockComment,
118    #[regex(r"[0-9]+(\.[0-9]+)?_?(mm|cm|m|inch|in|ft|yd|deg|rad|\?)?")]
119    #[regex(r"\.[0-9]+_?(mm|cm|m|inch|in|ft|yd|deg|rad|\?)?")]
120    Number,
121    #[token("..<")]
122    DoublePeriodLessThan,
123    #[token("..")]
124    DoublePeriod,
125    #[token("::")]
126    DoubleColon,
127    #[regex(r"[\p{Alphabetic}_][\p{Alphabetic}0-9_]*")]
128    Word,
129    #[token(">=")]
130    GtEq,
131    #[token("<=")]
132    LtEq,
133    #[token("==")]
134    EqEq,
135    #[token("=>")]
136    FatArrow,
137    #[token("!=")]
138    BangEq,
139    #[token("|>")]
140    PipeGt,
141    #[token("*")]
142    Star,
143    #[token("+")]
144    Plus,
145    #[token("-")]
146    Minus,
147    #[token("/")]
148    Slash,
149    #[token("%")]
150    Percent,
151    #[token("=")]
152    Eq,
153    #[token("<")]
154    Lt,
155    #[token(">")]
156    Gt,
157    #[token("\\")]
158    Backslash,
159    #[token("^")]
160    Caret,
161    #[token("||")]
162    PipePipe,
163    #[token("&&")]
164    AmpAmp,
165    #[token("|")]
166    Pipe,
167    #[token("&")]
168    Amp,
169    #[token("(")]
170    OpenParen,
171    #[token(")")]
172    CloseParen,
173    #[token("{")]
174    OpenBrace,
175    #[token("}")]
176    CloseBrace,
177    #[token("[")]
178    OpenBracket,
179    #[token("]")]
180    CloseBracket,
181    #[token("#")]
182    Hash,
183    #[token("!")]
184    Bang,
185    #[token("$")]
186    Dollar,
187    #[token(",")]
188    Comma,
189    #[token(":")]
190    Colon,
191    #[token(".")]
192    Period,
193    #[token("?")]
194    QuestionMark,
195    #[token("@")]
196    At,
197    #[token(";")]
198    SemiColon,
199}
200
201/// Lexes KCL source into a lossless token sequence.
202///
203/// This function does not return lexical errors separately. Invalid or
204/// recoverable input is preserved in the token stream with recovery token kinds.
205pub fn lex(source: &str) -> LexedSource<'_> {
206    let mut lexer = RawTokenKind::lexer(source);
207    let mut tokens = Vec::new();
208
209    while let Some(raw_kind) = lexer.next() {
210        let range = lexer.span();
211        let text = &source[range.clone()];
212        let kind = match raw_kind {
213            Ok(RawTokenKind::Whitespace) => SyntaxKind::Whitespace,
214            Ok(RawTokenKind::String) => SyntaxKind::String,
215            Ok(RawTokenKind::UnterminatedString) => SyntaxKind::UnterminatedString,
216            Ok(RawTokenKind::LineComment) => SyntaxKind::LineComment,
217            Ok(RawTokenKind::BlockComment) if text.ends_with("*/") => SyntaxKind::BlockComment,
218            Ok(RawTokenKind::BlockComment) => SyntaxKind::UnterminatedBlockComment,
219            Ok(RawTokenKind::Number) => SyntaxKind::Number,
220            Ok(RawTokenKind::DoublePeriodLessThan) => SyntaxKind::DoublePeriodLessThan,
221            Ok(RawTokenKind::DoublePeriod) => SyntaxKind::DoublePeriod,
222            Ok(RawTokenKind::DoubleColon) => SyntaxKind::DoubleColon,
223            Ok(RawTokenKind::Word) => keyword_or_word(text),
224            Ok(RawTokenKind::GtEq) => SyntaxKind::GtEq,
225            Ok(RawTokenKind::LtEq) => SyntaxKind::LtEq,
226            Ok(RawTokenKind::EqEq) => SyntaxKind::EqEq,
227            Ok(RawTokenKind::FatArrow) => SyntaxKind::FatArrow,
228            Ok(RawTokenKind::BangEq) => SyntaxKind::BangEq,
229            Ok(RawTokenKind::PipeGt) => SyntaxKind::PipeGt,
230            Ok(RawTokenKind::Star) => SyntaxKind::Star,
231            Ok(RawTokenKind::Plus) => SyntaxKind::Plus,
232            Ok(RawTokenKind::Minus) => SyntaxKind::Minus,
233            Ok(RawTokenKind::Slash) => SyntaxKind::Slash,
234            Ok(RawTokenKind::Percent) => SyntaxKind::Percent,
235            Ok(RawTokenKind::Eq) => SyntaxKind::Eq,
236            Ok(RawTokenKind::Lt) => SyntaxKind::Lt,
237            Ok(RawTokenKind::Gt) => SyntaxKind::Gt,
238            Ok(RawTokenKind::Backslash) => SyntaxKind::Backslash,
239            Ok(RawTokenKind::Caret) => SyntaxKind::Caret,
240            Ok(RawTokenKind::PipePipe) => SyntaxKind::PipePipe,
241            Ok(RawTokenKind::AmpAmp) => SyntaxKind::AmpAmp,
242            Ok(RawTokenKind::Pipe) => SyntaxKind::Pipe,
243            Ok(RawTokenKind::Amp) => SyntaxKind::Amp,
244            Ok(RawTokenKind::OpenParen) => SyntaxKind::OpenParen,
245            Ok(RawTokenKind::CloseParen) => SyntaxKind::CloseParen,
246            Ok(RawTokenKind::OpenBrace) => SyntaxKind::OpenBrace,
247            Ok(RawTokenKind::CloseBrace) => SyntaxKind::CloseBrace,
248            Ok(RawTokenKind::OpenBracket) => SyntaxKind::OpenBracket,
249            Ok(RawTokenKind::CloseBracket) => SyntaxKind::CloseBracket,
250            Ok(RawTokenKind::Hash) => SyntaxKind::Hash,
251            Ok(RawTokenKind::Bang) => SyntaxKind::Bang,
252            Ok(RawTokenKind::Dollar) => SyntaxKind::Dollar,
253            Ok(RawTokenKind::Comma) => SyntaxKind::Comma,
254            Ok(RawTokenKind::Colon) => SyntaxKind::Colon,
255            Ok(RawTokenKind::Period) => SyntaxKind::Period,
256            Ok(RawTokenKind::QuestionMark) => SyntaxKind::QuestionMark,
257            Ok(RawTokenKind::At) => SyntaxKind::At,
258            Ok(RawTokenKind::SemiColon) => SyntaxKind::SemiColon,
259            Err(()) => SyntaxKind::Unknown,
260        };
261        tokens.push(Token { kind, text, range });
262    }
263
264    LexedSource { source, tokens }
265}
266
267fn keyword_or_word(text: &str) -> SyntaxKind {
268    match text {
269        "if" => SyntaxKind::IfKw,
270        "else" => SyntaxKind::ElseKw,
271        "for" => SyntaxKind::ForKw,
272        "while" => SyntaxKind::WhileKw,
273        "return" => SyntaxKind::ReturnKw,
274        "break" => SyntaxKind::BreakKw,
275        "continue" => SyntaxKind::ContinueKw,
276        "fn" => SyntaxKind::FnKw,
277        "let" => SyntaxKind::LetKw,
278        "mut" => SyntaxKind::MutKw,
279        "as" => SyntaxKind::AsKw,
280        "loop" => SyntaxKind::LoopKw,
281        "true" => SyntaxKind::TrueKw,
282        "false" => SyntaxKind::FalseKw,
283        "nil" => SyntaxKind::NilKw,
284        "and" => SyntaxKind::AndKw,
285        "or" => SyntaxKind::OrKw,
286        "not" => SyntaxKind::NotKw,
287        "var" => SyntaxKind::VarKw,
288        "const" => SyntaxKind::ConstKw,
289        "import" => SyntaxKind::ImportKw,
290        "export" => SyntaxKind::ExportKw,
291        "type" => SyntaxKind::TypeKw,
292        "interface" => SyntaxKind::InterfaceKw,
293        "new" => SyntaxKind::NewKw,
294        "self" => SyntaxKind::SelfKw,
295        "record" => SyntaxKind::RecordKw,
296        "struct" => SyntaxKind::StructKw,
297        "object" => SyntaxKind::ObjectKw,
298        _ => SyntaxKind::Word,
299    }
300}
301
302fn block_comment(lexer: &mut logos::Lexer<'_, RawTokenKind>) {
303    if let Some(end) = lexer.remainder().find("*/") {
304        lexer.bump(end + 2);
305    } else {
306        lexer.bump(lexer.remainder().len());
307    }
308}
309
310fn unterminated_string(lexer: &mut logos::Lexer<'_, RawTokenKind>) {
311    let until_line_end = lexer
312        .remainder()
313        .find(['\n', '\r'])
314        .unwrap_or_else(|| lexer.remainder().len());
315    lexer.bump(until_line_end);
316}
kcl_syntax/lexer.rs

kcl_syntax/
lexer.rs
