kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::env;
5use std::fmt;
6use std::iter::Enumerate;
7use std::num::NonZeroUsize;
8use std::str::FromStr;
9
10use anyhow::Result;
11use kcl_error::KclErrorDetails;
12use parse_display::Display;
13use serde::Deserialize;
14use serde::Serialize;
15use tower_lsp::lsp_types::SemanticTokenType;
16use winnow::stream::ContainsToken;
17use winnow::stream::Stream;
18use winnow::{self};
19
20use crate::CompilationIssue;
21use crate::ModuleId;
22use crate::RuntimeFlag;
23use crate::SourceRange;
24use crate::errors::KclError;
25use crate::kcl_runtime_flags;
26use crate::parsing::ast::types::ItemVisibility;
27use crate::parsing::ast::types::VariableKind;
28
29mod tokeniser;
30
31pub(crate) mod adapter;
32
33#[cfg(test)]
34mod compat_tests;
35
36#[cfg(test)]
37mod error_matrix_tests;
38
39pub(crate) use tokeniser::RESERVED_SKETCH_BLOCK_WORDS;
40pub(crate) use tokeniser::RESERVED_WORDS;
41
42// Note the ordering, it's important that `m` comes after `mm` and `cm`.
43pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
44
45#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
46#[repr(u32)]
47pub enum NumericSuffix {
48    None,
49    Count,
50    Length,
51    Angle,
52    Mm,
53    Cm,
54    M,
55    Inch,
56    Ft,
57    Yd,
58    Deg,
59    Rad,
60    Unknown,
61}
62
63impl NumericSuffix {
64    #[allow(dead_code)]
65    pub fn is_none(self) -> bool {
66        self == Self::None
67    }
68
69    pub fn is_some(self) -> bool {
70        self != Self::None
71    }
72
73    pub fn digestable_id(&self) -> &[u8] {
74        match self {
75            NumericSuffix::None => &[],
76            NumericSuffix::Count => b"_",
77            NumericSuffix::Unknown => b"?",
78            NumericSuffix::Length => b"Length",
79            NumericSuffix::Angle => b"Angle",
80            NumericSuffix::Mm => b"mm",
81            NumericSuffix::Cm => b"cm",
82            NumericSuffix::M => b"m",
83            NumericSuffix::Inch => b"in",
84            NumericSuffix::Ft => b"ft",
85            NumericSuffix::Yd => b"yd",
86            NumericSuffix::Deg => b"deg",
87            NumericSuffix::Rad => b"rad",
88        }
89    }
90}
91
92impl FromStr for NumericSuffix {
93    type Err = CompilationIssue;
94
95    fn from_str(s: &str) -> Result<Self, Self::Err> {
96        match s {
97            "_" | "Count" => Ok(NumericSuffix::Count),
98            "Length" => Ok(NumericSuffix::Length),
99            "Angle" => Ok(NumericSuffix::Angle),
100            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
101            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
102            "m" | "meters" => Ok(NumericSuffix::M),
103            "inch" | "in" => Ok(NumericSuffix::Inch),
104            "ft" | "feet" => Ok(NumericSuffix::Ft),
105            "yd" | "yards" => Ok(NumericSuffix::Yd),
106            "deg" | "degrees" => Ok(NumericSuffix::Deg),
107            "rad" | "radians" => Ok(NumericSuffix::Rad),
108            "?" => Ok(NumericSuffix::Unknown),
109            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
110        }
111    }
112}
113
114impl fmt::Display for NumericSuffix {
115    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
116        match self {
117            NumericSuffix::None => Ok(()),
118            NumericSuffix::Count => write!(f, "_"),
119            NumericSuffix::Unknown => write!(f, "_?"),
120            NumericSuffix::Length => write!(f, "Length"),
121            NumericSuffix::Angle => write!(f, "Angle"),
122            NumericSuffix::Mm => write!(f, "mm"),
123            NumericSuffix::Cm => write!(f, "cm"),
124            NumericSuffix::M => write!(f, "m"),
125            NumericSuffix::Inch => write!(f, "in"),
126            NumericSuffix::Ft => write!(f, "ft"),
127            NumericSuffix::Yd => write!(f, "yd"),
128            NumericSuffix::Deg => write!(f, "deg"),
129            NumericSuffix::Rad => write!(f, "rad"),
130        }
131    }
132}
133
134#[derive(Clone, Debug, PartialEq)]
135pub(crate) struct TokenStream {
136    tokens: Vec<Token>,
137}
138
139impl TokenStream {
140    fn new(tokens: Vec<Token>) -> Self {
141        Self { tokens }
142    }
143
144    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
145        let tokens = std::mem::take(&mut self.tokens);
146        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
147            .into_iter()
148            .partition(|token| token.token_type != TokenType::Unknown);
149        self.tokens = tokens;
150        unknown_tokens
151    }
152
153    pub fn iter(&self) -> impl Iterator<Item = &Token> {
154        self.tokens.iter()
155    }
156
157    pub fn is_empty(&self) -> bool {
158        self.tokens.is_empty()
159    }
160
161    pub fn as_slice(&self) -> TokenSlice<'_> {
162        TokenSlice::from(self)
163    }
164}
165
166impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
167    fn from(stream: &'a TokenStream) -> Self {
168        TokenSlice {
169            start: 0,
170            end: stream.tokens.len(),
171            stream,
172        }
173    }
174}
175
176impl IntoIterator for TokenStream {
177    type Item = Token;
178
179    type IntoIter = std::vec::IntoIter<Token>;
180
181    fn into_iter(self) -> Self::IntoIter {
182        self.tokens.into_iter()
183    }
184}
185
186#[derive(Debug, Clone)]
187pub(crate) struct TokenSlice<'a> {
188    stream: &'a TokenStream,
189    /// Current position of the leading Token in the stream
190    start: usize,
191    /// The number of total Tokens in the stream
192    end: usize,
193}
194
195impl<'a> std::ops::Deref for TokenSlice<'a> {
196    type Target = [Token];
197
198    fn deref(&self) -> &Self::Target {
199        &self.stream.tokens[self.start..self.end]
200    }
201}
202
203impl<'a> TokenSlice<'a> {
204    pub fn token(&self, i: usize) -> &Token {
205        &self.stream.tokens[i + self.start]
206    }
207
208    pub fn iter(&self) -> impl Iterator<Item = &Token> {
209        (**self).iter()
210    }
211
212    pub fn without_ends(&self) -> Self {
213        Self {
214            start: self.start + 1,
215            end: self.end - 1,
216            stream: self.stream,
217        }
218    }
219
220    pub fn as_source_range(&self) -> SourceRange {
221        let stream_len = self.stream.tokens.len();
222        let first_token = if stream_len == self.start {
223            &self.stream.tokens[self.start - 1]
224        } else {
225            self.token(0)
226        };
227        let last_token = if stream_len == self.end {
228            &self.stream.tokens[stream_len - 1]
229        } else {
230            self.token(self.end - self.start)
231        };
232        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
233    }
234}
235
236impl<'a> IntoIterator for TokenSlice<'a> {
237    type Item = &'a Token;
238
239    type IntoIter = std::slice::Iter<'a, Token>;
240
241    fn into_iter(self) -> Self::IntoIter {
242        self.stream.tokens[self.start..self.end].iter()
243    }
244}
245
246impl<'a> Stream for TokenSlice<'a> {
247    type Token = Token;
248    type Slice = Self;
249    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
250    type Checkpoint = Checkpoint;
251
252    fn iter_offsets(&self) -> Self::IterOffsets {
253        #[allow(clippy::unnecessary_to_owned)]
254        self.to_vec().into_iter().enumerate()
255    }
256
257    fn eof_offset(&self) -> usize {
258        self.len()
259    }
260
261    fn next_token(&mut self) -> Option<Self::Token> {
262        let token = self.first()?.clone();
263        self.start += 1;
264        Some(token)
265    }
266
267    /// Split off the next token from the input
268    fn peek_token(&self) -> Option<Self::Token> {
269        Some(self.first()?.clone())
270    }
271
272    fn offset_for<P>(&self, predicate: P) -> Option<usize>
273    where
274        P: Fn(Self::Token) -> bool,
275    {
276        self.iter().position(|b| predicate(b.clone()))
277    }
278
279    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
280        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
281            Err(winnow::error::Needed::Size(needed))
282        } else {
283            Ok(tokens)
284        }
285    }
286
287    fn next_slice(&mut self, offset: usize) -> Self::Slice {
288        assert!(self.start + offset <= self.end);
289
290        let next = TokenSlice {
291            stream: self.stream,
292            start: self.start,
293            end: self.start + offset,
294        };
295        self.start += offset;
296        next
297    }
298
299    /// Split off a slice of tokens from the input
300    fn peek_slice(&self, offset: usize) -> Self::Slice {
301        assert!(self.start + offset <= self.end);
302
303        TokenSlice {
304            stream: self.stream,
305            start: self.start,
306            end: self.start + offset,
307        }
308    }
309
310    fn checkpoint(&self) -> Self::Checkpoint {
311        Checkpoint(self.start, self.end)
312    }
313
314    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
315        self.start = checkpoint.0;
316        self.end = checkpoint.1;
317    }
318
319    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
320        write!(f, "{self:?}")
321    }
322}
323
324impl<'a> winnow::stream::Offset for TokenSlice<'a> {
325    fn offset_from(&self, start: &Self) -> usize {
326        self.start - start.start
327    }
328}
329
330impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
331    fn offset_from(&self, start: &Checkpoint) -> usize {
332        self.start - start.0
333    }
334}
335
336impl winnow::stream::Offset for Checkpoint {
337    fn offset_from(&self, start: &Self) -> usize {
338        self.0 - start.0
339    }
340}
341
342impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
343    type PartialState = ();
344
345    fn complete(&mut self) -> Self::PartialState {}
346
347    fn restore_partial(&mut self, _: Self::PartialState) {}
348
349    fn is_partial_supported() -> bool {
350        false
351    }
352}
353
354impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
355    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
356        self.iter()
357            .enumerate()
358            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
359    }
360}
361
362#[derive(Clone, Debug)]
363pub struct Checkpoint(usize, usize);
364
365/// The types of tokens.
366#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
367#[display(style = "camelCase")]
368pub enum TokenType {
369    /// A number.
370    Number,
371    /// A word.
372    Word,
373    /// An operator.
374    Operator,
375    /// A string.
376    String,
377    /// A keyword.
378    Keyword,
379    /// A type.
380    Type,
381    /// A brace.
382    Brace,
383    /// A hash.
384    Hash,
385    /// A bang.
386    Bang,
387    /// A dollar sign.
388    Dollar,
389    /// Whitespace.
390    Whitespace,
391    /// A comma.
392    Comma,
393    /// A colon.
394    Colon,
395    /// A double colon: `::`
396    DoubleColon,
397    /// A period.
398    Period,
399    /// A double period: `..`.
400    DoublePeriod,
401    /// A double period and a less than: `..<`.
402    DoublePeriodLessThan,
403    /// A line comment.
404    LineComment,
405    /// A block comment.
406    BlockComment,
407    /// A function name.
408    Function,
409    /// Unknown lexemes.
410    Unknown,
411    /// The ? symbol, used for optional values.
412    QuestionMark,
413    /// The @ symbol.
414    At,
415    /// `;`
416    SemiColon,
417}
418
419/// Most KCL tokens correspond to LSP semantic tokens (but not all).
420impl TryFrom<TokenType> for SemanticTokenType {
421    type Error = anyhow::Error;
422    fn try_from(token_type: TokenType) -> Result<Self> {
423        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
424        // in the LSP implementation.
425        Ok(match token_type {
426            TokenType::Number => Self::NUMBER,
427            TokenType::Word => Self::VARIABLE,
428            TokenType::Keyword => Self::KEYWORD,
429            TokenType::Type => Self::TYPE,
430            TokenType::Operator => Self::OPERATOR,
431            TokenType::QuestionMark => Self::OPERATOR,
432            TokenType::String => Self::STRING,
433            TokenType::Bang => Self::OPERATOR,
434            TokenType::LineComment => Self::COMMENT,
435            TokenType::BlockComment => Self::COMMENT,
436            TokenType::Function => Self::FUNCTION,
437            TokenType::Whitespace
438            | TokenType::Brace
439            | TokenType::Comma
440            | TokenType::Colon
441            | TokenType::DoubleColon
442            | TokenType::Period
443            | TokenType::DoublePeriod
444            | TokenType::DoublePeriodLessThan
445            | TokenType::Hash
446            | TokenType::Dollar
447            | TokenType::At
448            | TokenType::SemiColon
449            | TokenType::Unknown => {
450                anyhow::bail!("unsupported token type: {:?}", token_type)
451            }
452        })
453    }
454}
455
456impl TokenType {
457    pub fn is_whitespace(&self) -> bool {
458        matches!(self, Self::Whitespace)
459    }
460
461    pub fn is_comment(&self) -> bool {
462        matches!(self, Self::LineComment | Self::BlockComment)
463    }
464}
465
466#[derive(Debug, PartialEq, Eq, Clone)]
467pub struct Token {
468    pub token_type: TokenType,
469    /// Offset in the source code where this token begins.
470    pub start: usize,
471    /// Offset in the source code where this token ends.
472    pub end: usize,
473    pub(super) module_id: ModuleId,
474    pub(super) value: String,
475}
476
477impl ContainsToken<Token> for (TokenType, &str) {
478    fn contains_token(&self, token: Token) -> bool {
479        self.0 == token.token_type && self.1 == token.value
480    }
481}
482
483impl ContainsToken<Token> for TokenType {
484    fn contains_token(&self, token: Token) -> bool {
485        *self == token.token_type
486    }
487}
488
489impl Token {
490    pub fn from_range(
491        range: std::ops::Range<usize>,
492        module_id: ModuleId,
493        token_type: TokenType,
494        value: String,
495    ) -> Self {
496        Self {
497            start: range.start,
498            end: range.end,
499            module_id,
500            value,
501            token_type,
502        }
503    }
504    pub fn is_code_token(&self) -> bool {
505        !matches!(
506            self.token_type,
507            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
508        )
509    }
510
511    pub fn as_source_range(&self) -> SourceRange {
512        SourceRange::new(self.start, self.end, self.module_id)
513    }
514
515    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
516        vec![self.as_source_range()]
517    }
518
519    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
520        if !matches!(self.token_type, TokenType::Keyword) {
521            return None;
522        }
523        match self.value.as_str() {
524            "export" => Some(ItemVisibility::Export),
525            _ => None,
526        }
527    }
528
529    pub fn numeric_value(&self) -> Option<f64> {
530        if self.token_type != TokenType::Number {
531            return None;
532        }
533        let value = &self.value;
534        let value = value
535            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
536            .map(|(s, _)| s)
537            .unwrap_or(value);
538        value.parse().ok()
539    }
540
541    pub fn uint_value(&self) -> Option<u32> {
542        if self.token_type != TokenType::Number {
543            return None;
544        }
545        let value = &self.value;
546        let value = value
547            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
548            .map(|(s, _)| s)
549            .unwrap_or(value);
550        value.parse().ok()
551    }
552
553    pub fn numeric_suffix(&self) -> NumericSuffix {
554        if self.token_type != TokenType::Number {
555            return NumericSuffix::None;
556        }
557
558        if self.value.ends_with('_') {
559            return NumericSuffix::Count;
560        }
561
562        for suffix in NUM_SUFFIXES {
563            if self.value.ends_with(suffix) {
564                return suffix.parse().unwrap();
565            }
566        }
567
568        NumericSuffix::None
569    }
570
571    /// Is this token the beginning of a variable/function declaration?
572    /// If so, what kind?
573    /// If not, returns None.
574    pub fn declaration_keyword(&self) -> Option<VariableKind> {
575        if !matches!(self.token_type, TokenType::Keyword) {
576            return None;
577        }
578        Some(match self.value.as_str() {
579            "fn" => VariableKind::Fn,
580            "var" | "let" | "const" => VariableKind::Const,
581            _ => return None,
582        })
583    }
584}
585
586impl From<Token> for SourceRange {
587    fn from(token: Token) -> Self {
588        Self::new(token.start, token.end, token.module_id)
589    }
590}
591
592impl From<&Token> for SourceRange {
593    fn from(token: &Token) -> Self {
594        Self::new(token.start, token.end, token.module_id)
595    }
596}
597
598/// Environment variable selecting which lexer implementation [`lex`] uses.
599pub(crate) const KCL_LEXER_ENV_VAR: &str = "KCL_LEXER";
600
601/// Which lexer implementation [`lex`] uses: the old winnow `tokeniser` (`Old`) or
602/// the new `kcl-syntax` logos lexer (`New`). Selected at runtime via the
603/// `KCL_LEXER` environment variable, so a process can pick either lexer without a
604/// rebuild.
605///
606/// Precedence: runtime flags > test override > `KCL_LEXER` >
607/// [`LexerMode::DEFAULT`].
608#[derive(Debug, Clone, Copy, PartialEq, Eq)]
609pub(crate) enum LexerMode {
610    Old,
611    New,
612}
613
614impl LexerMode {
615    /// The mode used when `KCL_LEXER` is unset.
616    const DEFAULT: Self = Self::Old;
617
618    /// Resolve the active lexer mode (see precedence on [`LexerMode`]).
619    pub(crate) fn resolve() -> Self {
620        let env_value = match env::var(KCL_LEXER_ENV_VAR) {
621            Ok(value) => Some(value),
622            Err(env::VarError::NotPresent) => None,
623            Err(env::VarError::NotUnicode(value)) => {
624                // Invalid-unicode env var: warn and fall back rather than crash.
625                Self::warn_once(|| {
626                    format!(
627                        "{KCL_LEXER_ENV_VAR} must be valid unicode; got `{}`. Defaulting to `old`.",
628                        value.to_string_lossy()
629                    )
630                });
631                None
632            }
633        };
634
635        Self::resolve_from_sources(
636            kcl_runtime_flags().use_new_lexer_parser,
637            Self::test_override_for_resolve(),
638            env_value.as_deref(),
639        )
640    }
641
642    fn resolve_from_sources(runtime_flag: RuntimeFlag, test_override: Option<Self>, env_value: Option<&str>) -> Self {
643        match runtime_flag {
644            RuntimeFlag::On => return Self::New,
645            RuntimeFlag::Off => return Self::Old,
646            RuntimeFlag::Unset => {}
647        }
648
649        if let Some(mode) = test_override {
650            return mode;
651        }
652
653        env_value.map(Self::parse).unwrap_or(Self::DEFAULT)
654    }
655
656    #[cfg(test)]
657    fn test_override_for_resolve() -> Option<Self> {
658        Self::test_override()
659    }
660
661    #[cfg(not(test))]
662    fn test_override_for_resolve() -> Option<Self> {
663        None
664    }
665
666    fn parse(value: &str) -> Self {
667        let value = value.trim();
668        if value.eq_ignore_ascii_case("old") {
669            return Self::Old;
670        }
671        if value.eq_ignore_ascii_case("new") {
672            return Self::New;
673        }
674
675        // A mistyped `KCL_LEXER` should not crash the process: warn and fall back
676        // to the old lexer (the conservative choice for a misconfiguration).
677        Self::warn_once(|| {
678            format!("Unsupported {KCL_LEXER_ENV_VAR} value `{value}`; expected `old` or `new`. Defaulting to `old`.")
679        });
680        Self::Old
681    }
682
683    /// Emit a one-time configuration warning through `crate::log` (gated on
684    /// `ZOO_LOG`). `resolve`/`parse` run on every `lex`, so a misconfigured
685    /// `KCL_LEXER` must not warn -- or allocate the message -- on every call. One
686    /// guard suffices: only one kind of misconfiguration can occur per process,
687    /// since the env var holds a single value.
688    fn warn_once(make_message: impl FnOnce() -> String) {
689        static WARNED: std::sync::Once = std::sync::Once::new();
690        WARNED.call_once(|| crate::log::log(make_message()));
691    }
692
693    #[cfg(test)]
694    fn test_override_value(self) -> u8 {
695        match self {
696            Self::Old => 1,
697            Self::New => 2,
698        }
699    }
700
701    #[cfg(test)]
702    fn test_override() -> Option<Self> {
703        match TEST_LEXER_MODE_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
704            1 => Some(Self::Old),
705            2 => Some(Self::New),
706            _ => None,
707        }
708    }
709
710    /// Override the lexer mode for the lifetime of the returned guard.
711    ///
712    /// This uses a process-global atomic, so it is only race-free under test
713    /// runners that isolate tests in separate processes (e.g. `cargo nextest`).
714    /// Under in-process parallel `cargo test`, prefer driving the lexer with an
715    /// explicit mode; reserve this guard for dispatch/integration tests.
716    #[cfg(test)]
717    pub(crate) fn override_for_test(mode: Self) -> LexerModeOverrideGuard {
718        let previous = TEST_LEXER_MODE_OVERRIDE.swap(mode.test_override_value(), std::sync::atomic::Ordering::SeqCst);
719        LexerModeOverrideGuard { previous }
720    }
721}
722
723#[cfg(test)]
724static TEST_LEXER_MODE_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
725
726#[cfg(test)]
727pub(crate) struct LexerModeOverrideGuard {
728    previous: u8,
729}
730
731#[cfg(test)]
732impl Drop for LexerModeOverrideGuard {
733    fn drop(&mut self) {
734        TEST_LEXER_MODE_OVERRIDE.store(self.previous, std::sync::atomic::Ordering::SeqCst);
735    }
736}
737
738// `lex` dispatches on the runtime `LexerMode`. `Old` runs the winnow
739// `tokeniser`; `New` runs the `kcl-syntax` adapter and folds any fatal lexical
740// diagnostics into a single lexical `KclError`, preserving the public `Result`
741// contract. (The LSP consumes the richer `LexResult` directly so it can keep
742// tokens for highlighting while reporting diagnostics.)
743pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
744    match LexerMode::resolve() {
745        LexerMode::Old => lex_legacy(s, module_id),
746        LexerMode::New => {
747            let result = adapter::lex_with_diagnostics(s, module_id);
748            match result.to_lexical_error() {
749                Some(err) => Err(err),
750                None => Ok(result.tokens),
751            }
752        }
753    }
754}
755
756fn lex_legacy(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
757    tokeniser::lex(s, module_id).map_err(|err| {
758        let (input, offset): (Vec<char>, usize) = (err.input().chars().collect(), err.offset());
759        let module_id = err.input().state.module_id;
760
761        if offset >= input.len() {
762            // From the winnow docs:
763            //
764            // This is an offset, not an index, and may point to
765            // the end of input (input.len()) on eof errors.
766
767            return KclError::new_lexical(KclErrorDetails::new(
768                "unexpected EOF while parsing".to_owned(),
769                vec![SourceRange::new(offset, offset, module_id)],
770            ));
771        }
772
773        // TODO: Add the Winnow tokenizer context to the error.
774        // See https://github.com/KittyCAD/modeling-app/issues/784
775        let bad_token = &input[offset];
776        // TODO: Add the Winnow parser context to the error.
777        // See https://github.com/KittyCAD/modeling-app/issues/784
778        KclError::new_lexical(KclErrorDetails::new(
779            format!("found unknown token '{bad_token}'"),
780            vec![SourceRange::new(offset, offset + 1, module_id)],
781        ))
782    })
783}
784
785#[cfg(test)]
786mod lexer_mode_tests {
787    use super::LexerMode;
788    use super::lex;
789    use crate::KclRuntimeFlags;
790    use crate::ModuleId;
791    use crate::RuntimeFlag;
792
793    fn set_runtime_lexer_flag(flag: RuntimeFlag) {
794        crate::set_kcl_runtime_flags(KclRuntimeFlags {
795            use_new_lexer_parser: flag,
796        });
797    }
798
799    fn reset_runtime_lexer_flags() {
800        crate::set_kcl_runtime_flags(KclRuntimeFlags::DEFAULT);
801    }
802
803    #[test]
804    fn default_mode_is_old() {
805        reset_runtime_lexer_flags();
806        assert_eq!(LexerMode::DEFAULT, LexerMode::Old);
807    }
808
809    #[test]
810    fn parse_accepts_known_values_case_insensitively() {
811        assert_eq!(LexerMode::parse("old"), LexerMode::Old);
812        assert_eq!(LexerMode::parse("  NEW  "), LexerMode::New);
813    }
814
815    #[test]
816    fn parse_falls_back_to_old_on_unknown_value() {
817        // An unknown value warns and defaults to the old lexer instead of panicking.
818        assert_eq!(LexerMode::parse("rowan"), LexerMode::Old);
819    }
820
821    #[test]
822    fn override_guard_sets_and_restores_mode() {
823        reset_runtime_lexer_flags();
824        // Reserved for dispatch/integration tests; relies on the process-global
825        // atomic, which is race-free under nextest's process isolation.
826        {
827            let _guard = LexerMode::override_for_test(LexerMode::New);
828            assert_eq!(LexerMode::resolve(), LexerMode::New);
829        }
830        let _guard = LexerMode::override_for_test(LexerMode::Old);
831        assert_eq!(LexerMode::resolve(), LexerMode::Old);
832    }
833
834    #[test]
835    fn runtime_flags_default_to_unset() {
836        reset_runtime_lexer_flags();
837        assert_eq!(
838            crate::kcl_runtime_flags(),
839            KclRuntimeFlags {
840                use_new_lexer_parser: RuntimeFlag::Unset,
841            }
842        );
843    }
844
845    #[test]
846    fn runtime_flag_on_selects_new_lexer() {
847        reset_runtime_lexer_flags();
848        set_runtime_lexer_flag(RuntimeFlag::On);
849        assert_eq!(LexerMode::resolve(), LexerMode::New);
850    }
851
852    #[test]
853    fn runtime_flag_off_selects_old_lexer() {
854        reset_runtime_lexer_flags();
855        set_runtime_lexer_flag(RuntimeFlag::Off);
856        assert_eq!(LexerMode::resolve(), LexerMode::Old);
857    }
858
859    #[test]
860    fn runtime_flag_takes_priority_over_test_override_and_env() {
861        assert_eq!(
862            LexerMode::resolve_from_sources(RuntimeFlag::Off, Some(LexerMode::New), Some("new")),
863            LexerMode::Old
864        );
865        assert_eq!(
866            LexerMode::resolve_from_sources(RuntimeFlag::On, Some(LexerMode::Old), Some("old")),
867            LexerMode::New
868        );
869    }
870
871    #[test]
872    fn unset_runtime_flag_allows_env_to_select_lexer() {
873        assert_eq!(
874            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, Some("new")),
875            LexerMode::New
876        );
877        assert_eq!(
878            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, Some("old")),
879            LexerMode::Old
880        );
881    }
882
883    #[test]
884    fn unset_runtime_flag_and_missing_env_selects_default_lexer() {
885        assert_eq!(
886            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, None),
887            LexerMode::DEFAULT
888        );
889    }
890
891    #[test]
892    fn test_override_takes_priority_over_env() {
893        assert_eq!(
894            LexerMode::resolve_from_sources(RuntimeFlag::Unset, Some(LexerMode::Old), Some("new")),
895            LexerMode::Old
896        );
897        assert_eq!(
898            LexerMode::resolve_from_sources(RuntimeFlag::Unset, Some(LexerMode::New), Some("old")),
899            LexerMode::New
900        );
901    }
902
903    /// Exercises the `New` arm of `lex` in default CI: no `KCL_LEXER` env var is
904    /// set; the new lexer is selected via the process-global test override (which
905    /// is race-free under nextest's process-per-test isolation).
906    ///
907    /// The unterminated-string assertion is deliberately a *distinguishing* one:
908    /// the new lexer folds the recovery token into the message "unterminated
909    /// string literal", whereas the old lexer reports `found unknown token '"'`.
910    /// Asserting the new-lexer-only message proves `lex` took the `New` arm --
911    /// not merely that some lexer ran.
912    #[test]
913    fn lex_dispatches_to_new_lexer() {
914        reset_runtime_lexer_flags();
915        let _guard = LexerMode::override_for_test(LexerMode::New);
916        assert_eq!(LexerMode::resolve(), LexerMode::New);
917
918        let module_id = ModuleId::default();
919
920        // Valid input flows through the New arm and yields a token stream.
921        let tokens = lex("x = 1", module_id).expect("new lexer should tokenize valid input");
922        assert!(!tokens.is_empty(), "expected a non-empty token stream");
923
924        // Unterminated string: the new-lexer-only message (see doc comment).
925        let err = lex("\"abc", module_id).expect_err("unterminated string is a lexical error");
926        assert_eq!(err.error_type(), "lexical");
927        assert_eq!(err.message(), "unterminated string literal");
928    }
929}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
