kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::env;
5use std::fmt;
6use std::iter::Enumerate;
7use std::num::NonZeroUsize;
8use std::str::FromStr;
9
10use anyhow::Result;
11use kcl_error::KclErrorDetails;
12use parse_display::Display;
13use serde::Deserialize;
14use serde::Serialize;
15use tower_lsp::lsp_types::SemanticTokenType;
16use winnow::stream::ContainsToken;
17use winnow::stream::Stream;
18use winnow::{self};
19
20use crate::CompilationIssue;
21use crate::ModuleId;
22use crate::RuntimeFlag;
23use crate::SourceRange;
24use crate::errors::KclError;
25use crate::kcl_runtime_flags;
26use crate::parsing::ast::types::ItemVisibility;
27use crate::parsing::ast::types::VariableKind;
28use crate::runtime_flags::RuntimeFlagResolve;
29use crate::runtime_flags::resolve_from_sources;
30
31mod tokeniser;
32
33#[doc(hidden)]
34pub mod adapter;
35
36#[cfg(test)]
37mod compat_tests;
38
39#[cfg(test)]
40mod error_matrix_tests;
41
42pub(crate) use tokeniser::RESERVED_SKETCH_BLOCK_WORDS;
43pub use tokeniser::RESERVED_WORDS;
44
45// Note the ordering, it's important that `m` comes after `mm` and `cm`.
46pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
47
48#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
49#[repr(u32)]
50pub enum NumericSuffix {
51    None,
52    Count,
53    Length,
54    Angle,
55    Mm,
56    Cm,
57    M,
58    Inch,
59    Ft,
60    Yd,
61    Deg,
62    Rad,
63    Unknown,
64}
65
66impl NumericSuffix {
67    #[allow(dead_code)]
68    pub fn is_none(self) -> bool {
69        self == Self::None
70    }
71
72    pub fn is_some(self) -> bool {
73        self != Self::None
74    }
75
76    pub fn digestable_id(&self) -> &[u8] {
77        match self {
78            NumericSuffix::None => &[],
79            NumericSuffix::Count => b"_",
80            NumericSuffix::Unknown => b"?",
81            NumericSuffix::Length => b"Length",
82            NumericSuffix::Angle => b"Angle",
83            NumericSuffix::Mm => b"mm",
84            NumericSuffix::Cm => b"cm",
85            NumericSuffix::M => b"m",
86            NumericSuffix::Inch => b"in",
87            NumericSuffix::Ft => b"ft",
88            NumericSuffix::Yd => b"yd",
89            NumericSuffix::Deg => b"deg",
90            NumericSuffix::Rad => b"rad",
91        }
92    }
93}
94
95impl FromStr for NumericSuffix {
96    type Err = CompilationIssue;
97
98    fn from_str(s: &str) -> Result<Self, Self::Err> {
99        match s {
100            "_" | "Count" => Ok(NumericSuffix::Count),
101            "Length" => Ok(NumericSuffix::Length),
102            "Angle" => Ok(NumericSuffix::Angle),
103            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
104            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
105            "m" | "meters" => Ok(NumericSuffix::M),
106            "inch" | "in" => Ok(NumericSuffix::Inch),
107            "ft" | "feet" => Ok(NumericSuffix::Ft),
108            "yd" | "yards" => Ok(NumericSuffix::Yd),
109            "deg" | "degrees" => Ok(NumericSuffix::Deg),
110            "rad" | "radians" => Ok(NumericSuffix::Rad),
111            "?" => Ok(NumericSuffix::Unknown),
112            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
113        }
114    }
115}
116
117impl fmt::Display for NumericSuffix {
118    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
119        match self {
120            NumericSuffix::None => Ok(()),
121            NumericSuffix::Count => write!(f, "_"),
122            NumericSuffix::Unknown => write!(f, "_?"),
123            NumericSuffix::Length => write!(f, "Length"),
124            NumericSuffix::Angle => write!(f, "Angle"),
125            NumericSuffix::Mm => write!(f, "mm"),
126            NumericSuffix::Cm => write!(f, "cm"),
127            NumericSuffix::M => write!(f, "m"),
128            NumericSuffix::Inch => write!(f, "in"),
129            NumericSuffix::Ft => write!(f, "ft"),
130            NumericSuffix::Yd => write!(f, "yd"),
131            NumericSuffix::Deg => write!(f, "deg"),
132            NumericSuffix::Rad => write!(f, "rad"),
133        }
134    }
135}
136
137#[derive(Clone, Debug, PartialEq)]
138pub struct TokenStream {
139    tokens: Vec<Token>,
140}
141
142impl TokenStream {
143    fn new(tokens: Vec<Token>) -> Self {
144        Self { tokens }
145    }
146
147    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
148        let tokens = std::mem::take(&mut self.tokens);
149        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
150            .into_iter()
151            .partition(|token| token.token_type != TokenType::Unknown);
152        self.tokens = tokens;
153        unknown_tokens
154    }
155
156    pub fn iter(&self) -> impl Iterator<Item = &Token> {
157        self.tokens.iter()
158    }
159
160    pub fn is_empty(&self) -> bool {
161        self.tokens.is_empty()
162    }
163
164    pub fn as_slice(&self) -> TokenSlice<'_> {
165        TokenSlice::from(self)
166    }
167}
168
169impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
170    fn from(stream: &'a TokenStream) -> Self {
171        TokenSlice {
172            start: 0,
173            end: stream.tokens.len(),
174            stream,
175        }
176    }
177}
178
179impl IntoIterator for TokenStream {
180    type Item = Token;
181
182    type IntoIter = std::vec::IntoIter<Token>;
183
184    fn into_iter(self) -> Self::IntoIter {
185        self.tokens.into_iter()
186    }
187}
188
189#[derive(Debug, Clone)]
190pub struct TokenSlice<'a> {
191    stream: &'a TokenStream,
192    /// Current position of the leading Token in the stream
193    start: usize,
194    /// The number of total Tokens in the stream
195    end: usize,
196}
197
198impl<'a> std::ops::Deref for TokenSlice<'a> {
199    type Target = [Token];
200
201    fn deref(&self) -> &Self::Target {
202        &self.stream.tokens[self.start..self.end]
203    }
204}
205
206impl<'a> TokenSlice<'a> {
207    pub fn token(&self, i: usize) -> &Token {
208        &self.stream.tokens[i + self.start]
209    }
210
211    pub fn iter(&self) -> impl Iterator<Item = &Token> {
212        (**self).iter()
213    }
214
215    pub fn without_ends(&self) -> Self {
216        Self {
217            start: self.start + 1,
218            end: self.end - 1,
219            stream: self.stream,
220        }
221    }
222
223    pub fn as_source_range(&self) -> SourceRange {
224        let stream_len = self.stream.tokens.len();
225        let first_token = if stream_len == self.start {
226            &self.stream.tokens[self.start - 1]
227        } else {
228            self.token(0)
229        };
230        let last_token = if stream_len == self.end {
231            &self.stream.tokens[stream_len - 1]
232        } else {
233            self.token(self.end - self.start)
234        };
235        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
236    }
237}
238
239impl<'a> IntoIterator for TokenSlice<'a> {
240    type Item = &'a Token;
241
242    type IntoIter = std::slice::Iter<'a, Token>;
243
244    fn into_iter(self) -> Self::IntoIter {
245        self.stream.tokens[self.start..self.end].iter()
246    }
247}
248
249impl<'a> Stream for TokenSlice<'a> {
250    type Token = Token;
251    type Slice = Self;
252    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
253    type Checkpoint = Checkpoint;
254
255    fn iter_offsets(&self) -> Self::IterOffsets {
256        #[allow(clippy::unnecessary_to_owned)]
257        self.to_vec().into_iter().enumerate()
258    }
259
260    fn eof_offset(&self) -> usize {
261        self.len()
262    }
263
264    fn next_token(&mut self) -> Option<Self::Token> {
265        let token = self.first()?.clone();
266        self.start += 1;
267        Some(token)
268    }
269
270    /// Split off the next token from the input
271    fn peek_token(&self) -> Option<Self::Token> {
272        Some(self.first()?.clone())
273    }
274
275    fn offset_for<P>(&self, predicate: P) -> Option<usize>
276    where
277        P: Fn(Self::Token) -> bool,
278    {
279        self.iter().position(|b| predicate(b.clone()))
280    }
281
282    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
283        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
284            Err(winnow::error::Needed::Size(needed))
285        } else {
286            Ok(tokens)
287        }
288    }
289
290    fn next_slice(&mut self, offset: usize) -> Self::Slice {
291        assert!(self.start + offset <= self.end);
292
293        let next = TokenSlice {
294            stream: self.stream,
295            start: self.start,
296            end: self.start + offset,
297        };
298        self.start += offset;
299        next
300    }
301
302    /// Split off a slice of tokens from the input
303    fn peek_slice(&self, offset: usize) -> Self::Slice {
304        assert!(self.start + offset <= self.end);
305
306        TokenSlice {
307            stream: self.stream,
308            start: self.start,
309            end: self.start + offset,
310        }
311    }
312
313    fn checkpoint(&self) -> Self::Checkpoint {
314        Checkpoint(self.start, self.end)
315    }
316
317    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
318        self.start = checkpoint.0;
319        self.end = checkpoint.1;
320    }
321
322    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
323        write!(f, "{self:?}")
324    }
325}
326
327impl<'a> winnow::stream::Offset for TokenSlice<'a> {
328    fn offset_from(&self, start: &Self) -> usize {
329        self.start - start.start
330    }
331}
332
333impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
334    fn offset_from(&self, start: &Checkpoint) -> usize {
335        self.start - start.0
336    }
337}
338
339impl winnow::stream::Offset for Checkpoint {
340    fn offset_from(&self, start: &Self) -> usize {
341        self.0 - start.0
342    }
343}
344
345impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
346    type PartialState = ();
347
348    fn complete(&mut self) -> Self::PartialState {}
349
350    fn restore_partial(&mut self, _: Self::PartialState) {}
351
352    fn is_partial_supported() -> bool {
353        false
354    }
355}
356
357impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
358    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
359        self.iter()
360            .enumerate()
361            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
362    }
363}
364
365#[derive(Clone, Debug)]
366pub struct Checkpoint(usize, usize);
367
368/// The types of tokens.
369#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
370#[display(style = "camelCase")]
371pub enum TokenType {
372    /// A number.
373    Number,
374    /// A word.
375    Word,
376    /// An operator.
377    Operator,
378    /// A string.
379    String,
380    /// A keyword.
381    Keyword,
382    /// A type.
383    Type,
384    /// A brace.
385    Brace,
386    /// A hash.
387    Hash,
388    /// A bang.
389    Bang,
390    /// A dollar sign.
391    Dollar,
392    /// Whitespace.
393    Whitespace,
394    /// A comma.
395    Comma,
396    /// A colon.
397    Colon,
398    /// A double colon: `::`
399    DoubleColon,
400    /// A period.
401    Period,
402    /// A double period: `..`.
403    DoublePeriod,
404    /// A double period and a less than: `..<`.
405    DoublePeriodLessThan,
406    /// A line comment.
407    LineComment,
408    /// A block comment.
409    BlockComment,
410    /// A function name.
411    Function,
412    /// Unknown lexemes.
413    Unknown,
414    /// The ? symbol, used for optional values.
415    QuestionMark,
416    /// The @ symbol.
417    At,
418    /// `;`
419    SemiColon,
420}
421
422/// Most KCL tokens correspond to LSP semantic tokens (but not all).
423impl TryFrom<TokenType> for SemanticTokenType {
424    type Error = anyhow::Error;
425    fn try_from(token_type: TokenType) -> Result<Self> {
426        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
427        // in the LSP implementation.
428        Ok(match token_type {
429            TokenType::Number => Self::NUMBER,
430            TokenType::Word => Self::VARIABLE,
431            TokenType::Keyword => Self::KEYWORD,
432            TokenType::Type => Self::TYPE,
433            TokenType::Operator => Self::OPERATOR,
434            TokenType::QuestionMark => Self::OPERATOR,
435            TokenType::String => Self::STRING,
436            TokenType::Bang => Self::OPERATOR,
437            TokenType::LineComment => Self::COMMENT,
438            TokenType::BlockComment => Self::COMMENT,
439            TokenType::Function => Self::FUNCTION,
440            TokenType::Whitespace
441            | TokenType::Brace
442            | TokenType::Comma
443            | TokenType::Colon
444            | TokenType::DoubleColon
445            | TokenType::Period
446            | TokenType::DoublePeriod
447            | TokenType::DoublePeriodLessThan
448            | TokenType::Hash
449            | TokenType::Dollar
450            | TokenType::At
451            | TokenType::SemiColon
452            | TokenType::Unknown => {
453                anyhow::bail!("unsupported token type: {:?}", token_type)
454            }
455        })
456    }
457}
458
459impl TokenType {
460    pub fn is_whitespace(&self) -> bool {
461        matches!(self, Self::Whitespace)
462    }
463
464    pub fn is_comment(&self) -> bool {
465        matches!(self, Self::LineComment | Self::BlockComment)
466    }
467}
468
469#[derive(Debug, PartialEq, Eq, Clone)]
470pub struct Token {
471    pub token_type: TokenType,
472    /// Offset in the source code where this token begins.
473    pub start: usize,
474    /// Offset in the source code where this token ends.
475    pub end: usize,
476    pub(super) module_id: ModuleId,
477    pub(super) value: String,
478}
479
480impl ContainsToken<Token> for (TokenType, &str) {
481    fn contains_token(&self, token: Token) -> bool {
482        self.0 == token.token_type && self.1 == token.value
483    }
484}
485
486impl ContainsToken<Token> for TokenType {
487    fn contains_token(&self, token: Token) -> bool {
488        *self == token.token_type
489    }
490}
491
492impl Token {
493    pub fn from_range(
494        range: std::ops::Range<usize>,
495        module_id: ModuleId,
496        token_type: TokenType,
497        value: String,
498    ) -> Self {
499        Self {
500            start: range.start,
501            end: range.end,
502            module_id,
503            value,
504            token_type,
505        }
506    }
507    pub fn is_code_token(&self) -> bool {
508        !matches!(
509            self.token_type,
510            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
511        )
512    }
513
514    pub fn as_source_range(&self) -> SourceRange {
515        SourceRange::new(self.start, self.end, self.module_id)
516    }
517
518    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
519        vec![self.as_source_range()]
520    }
521
522    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
523        if !matches!(self.token_type, TokenType::Keyword) {
524            return None;
525        }
526        match self.value.as_str() {
527            "export" => Some(ItemVisibility::Export),
528            _ => None,
529        }
530    }
531
532    pub fn numeric_value(&self) -> Option<f64> {
533        if self.token_type != TokenType::Number {
534            return None;
535        }
536        let value = &self.value;
537        let value = value
538            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
539            .map(|(s, _)| s)
540            .unwrap_or(value);
541        value.parse().ok()
542    }
543
544    pub fn uint_value(&self) -> Option<u32> {
545        if self.token_type != TokenType::Number {
546            return None;
547        }
548        let value = &self.value;
549        let value = value
550            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
551            .map(|(s, _)| s)
552            .unwrap_or(value);
553        value.parse().ok()
554    }
555
556    pub fn numeric_suffix(&self) -> NumericSuffix {
557        if self.token_type != TokenType::Number {
558            return NumericSuffix::None;
559        }
560
561        if self.value.ends_with('_') {
562            return NumericSuffix::Count;
563        }
564
565        for suffix in NUM_SUFFIXES {
566            if self.value.ends_with(suffix) {
567                return suffix.parse().unwrap();
568            }
569        }
570
571        NumericSuffix::None
572    }
573
574    /// Is this token the beginning of a variable/function declaration?
575    /// If so, what kind?
576    /// If not, returns None.
577    pub fn declaration_keyword(&self) -> Option<VariableKind> {
578        if !matches!(self.token_type, TokenType::Keyword) {
579            return None;
580        }
581        Some(match self.value.as_str() {
582            "fn" => VariableKind::Fn,
583            "var" | "let" | "const" => VariableKind::Const,
584            _ => return None,
585        })
586    }
587}
588
589impl From<Token> for SourceRange {
590    fn from(token: Token) -> Self {
591        Self::new(token.start, token.end, token.module_id)
592    }
593}
594
595impl From<&Token> for SourceRange {
596    fn from(token: &Token) -> Self {
597        Self::new(token.start, token.end, token.module_id)
598    }
599}
600
601/// Environment variable selecting which lexer implementation [`lex`] uses.
602pub(crate) const KCL_LEXER_ENV_VAR: &str = "KCL_LEXER";
603
604/// Which lexer implementation [`lex`] uses: the old winnow `tokeniser` (`Old`) or
605/// the new `kcl-syntax` logos lexer (`New`). Selected at runtime via the
606/// `KCL_LEXER` environment variable, so a process can pick either lexer without a
607/// rebuild.
608///
609/// Precedence: runtime flags > test override > `KCL_LEXER` >
610/// [`LexerMode::DEFAULT`].
611#[derive(Debug, Clone, Copy, PartialEq, Eq)]
612pub enum LexerMode {
613    Old,
614    New,
615}
616
617impl RuntimeFlagResolve for LexerMode {
618    fn on() -> Self {
619        Self::New
620    }
621
622    fn off() -> Self {
623        Self::Old
624    }
625
626    fn resolve_default() -> Self {
627        Self::DEFAULT
628    }
629
630    fn parse_env_var(value: &str) -> Self {
631        Self::parse(value)
632    }
633}
634
635impl LexerMode {
636    /// The mode used when `KCL_LEXER` is unset.
637    const DEFAULT: Self = Self::Old;
638
639    /// Resolve the active lexer mode (see precedence on [`LexerMode`]).
640    pub fn resolve() -> Self {
641        let env_value = match env::var(KCL_LEXER_ENV_VAR) {
642            Ok(value) => Some(value),
643            Err(env::VarError::NotPresent) => None,
644            Err(env::VarError::NotUnicode(value)) => {
645                // Invalid-unicode env var: warn and fall back rather than crash.
646                Self::warn_once(|| {
647                    format!(
648                        "{KCL_LEXER_ENV_VAR} must be valid unicode; got `{}`. Defaulting to `old`.",
649                        value.to_string_lossy()
650                    )
651                });
652                None
653            }
654        };
655
656        Self::resolve_from_sources(
657            kcl_runtime_flags().use_new_lexer_parser,
658            Self::test_override_for_resolve(),
659            env_value.as_deref(),
660        )
661    }
662
663    fn resolve_from_sources(runtime_flag: RuntimeFlag, test_override: Option<Self>, env_value: Option<&str>) -> Self {
664        resolve_from_sources(runtime_flag, test_override, env_value)
665    }
666
667    #[cfg(any(test, feature = "lsp-test-util"))]
668    fn test_override_for_resolve() -> Option<Self> {
669        Self::test_override()
670    }
671
672    #[cfg(not(any(test, feature = "lsp-test-util")))]
673    fn test_override_for_resolve() -> Option<Self> {
674        None
675    }
676
677    fn parse(value: &str) -> Self {
678        let value = value.trim();
679        if value.eq_ignore_ascii_case("old") {
680            return Self::Old;
681        }
682        if value.eq_ignore_ascii_case("new") {
683            return Self::New;
684        }
685
686        // A mistyped `KCL_LEXER` should not crash the process: warn and fall back
687        // to the old lexer (the conservative choice for a misconfiguration).
688        Self::warn_once(|| {
689            format!("Unsupported {KCL_LEXER_ENV_VAR} value `{value}`; expected `old` or `new`. Defaulting to `old`.")
690        });
691        Self::Old
692    }
693
694    /// Emit a one-time configuration warning through `crate::log` (gated on
695    /// `ZOO_LOG`). `resolve`/`parse` run on every `lex`, so a misconfigured
696    /// `KCL_LEXER` must not warn -- or allocate the message -- on every call. One
697    /// guard suffices: only one kind of misconfiguration can occur per process,
698    /// since the env var holds a single value.
699    fn warn_once(make_message: impl FnOnce() -> String) {
700        static WARNED: std::sync::Once = std::sync::Once::new();
701        WARNED.call_once(|| crate::log::log(make_message()));
702    }
703
704    #[cfg(any(test, feature = "lsp-test-util"))]
705    fn test_override_value(self) -> u8 {
706        match self {
707            Self::Old => 1,
708            Self::New => 2,
709        }
710    }
711
712    #[cfg(any(test, feature = "lsp-test-util"))]
713    fn test_override() -> Option<Self> {
714        match TEST_LEXER_MODE_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
715            1 => Some(Self::Old),
716            2 => Some(Self::New),
717            _ => None,
718        }
719    }
720
721    /// Override the lexer mode for the lifetime of the returned guard.
722    ///
723    /// This uses a process-global atomic, so it is only race-free under test
724    /// runners that isolate tests in separate processes (e.g. `cargo nextest`).
725    /// Under in-process parallel `cargo test`, prefer driving the lexer with an
726    /// explicit mode; reserve this guard for dispatch/integration tests.
727    #[cfg(any(test, feature = "lsp-test-util"))]
728    pub fn override_for_test(mode: Self) -> LexerModeOverrideGuard {
729        let previous = TEST_LEXER_MODE_OVERRIDE.swap(mode.test_override_value(), std::sync::atomic::Ordering::SeqCst);
730        LexerModeOverrideGuard { previous }
731    }
732}
733
734#[cfg(any(test, feature = "lsp-test-util"))]
735static TEST_LEXER_MODE_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
736
737#[cfg(any(test, feature = "lsp-test-util"))]
738pub struct LexerModeOverrideGuard {
739    previous: u8,
740}
741
742#[cfg(any(test, feature = "lsp-test-util"))]
743impl Drop for LexerModeOverrideGuard {
744    fn drop(&mut self) {
745        TEST_LEXER_MODE_OVERRIDE.store(self.previous, std::sync::atomic::Ordering::SeqCst);
746    }
747}
748
749// `lex` dispatches on the runtime `LexerMode`. `Old` runs the winnow
750// `tokeniser`; `New` runs the `kcl-syntax` adapter and folds any fatal lexical
751// diagnostics into a single lexical `KclError`, preserving the public `Result`
752// contract. (The LSP consumes the richer `LexResult` directly so it can keep
753// tokens for highlighting while reporting diagnostics.)
754pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
755    match LexerMode::resolve() {
756        LexerMode::Old => lex_legacy(s, module_id),
757        LexerMode::New => {
758            let result = adapter::lex_with_diagnostics(s, module_id);
759            match result.to_lexical_error() {
760                Some(err) => Err(err),
761                None => Ok(result.tokens),
762            }
763        }
764    }
765}
766
767fn lex_legacy(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
768    tokeniser::lex(s, module_id).map_err(|err| {
769        let (input, offset): (Vec<char>, usize) = (err.input().chars().collect(), err.offset());
770        let module_id = err.input().state.module_id;
771
772        if offset >= input.len() {
773            // From the winnow docs:
774            //
775            // This is an offset, not an index, and may point to
776            // the end of input (input.len()) on eof errors.
777
778            return KclError::new_lexical(KclErrorDetails::new(
779                "unexpected EOF while parsing".to_owned(),
780                vec![SourceRange::new(offset, offset, module_id)],
781            ));
782        }
783
784        // TODO: Add the Winnow tokenizer context to the error.
785        // See https://github.com/KittyCAD/modeling-app/issues/784
786        let bad_token = &input[offset];
787        // TODO: Add the Winnow parser context to the error.
788        // See https://github.com/KittyCAD/modeling-app/issues/784
789        KclError::new_lexical(KclErrorDetails::new(
790            format!("found unknown token '{bad_token}'"),
791            vec![SourceRange::new(offset, offset + 1, module_id)],
792        ))
793    })
794}
795
796#[cfg(test)]
797mod lexer_mode_tests {
798    use super::LexerMode;
799    use super::lex;
800    use crate::KclRuntimeFlags;
801    use crate::ModuleId;
802    use crate::RuntimeFlag;
803
804    fn set_runtime_lexer_flag(flag: RuntimeFlag) {
805        crate::set_kcl_runtime_flags(KclRuntimeFlags {
806            use_new_lexer_parser: flag,
807            ..Default::default()
808        });
809    }
810
811    fn reset_runtime_lexer_flags() {
812        crate::set_kcl_runtime_flags(KclRuntimeFlags::DEFAULT);
813    }
814
815    #[test]
816    fn default_mode_is_old() {
817        reset_runtime_lexer_flags();
818        assert_eq!(LexerMode::DEFAULT, LexerMode::Old);
819    }
820
821    #[test]
822    fn parse_accepts_known_values_case_insensitively() {
823        assert_eq!(LexerMode::parse("old"), LexerMode::Old);
824        assert_eq!(LexerMode::parse("  NEW  "), LexerMode::New);
825    }
826
827    #[test]
828    fn parse_falls_back_to_old_on_unknown_value() {
829        // An unknown value warns and defaults to the old lexer instead of panicking.
830        assert_eq!(LexerMode::parse("rowan"), LexerMode::Old);
831    }
832
833    #[test]
834    fn override_guard_sets_and_restores_mode() {
835        reset_runtime_lexer_flags();
836        // Reserved for dispatch/integration tests; relies on the process-global
837        // atomic, which is race-free under nextest's process isolation.
838        {
839            let _guard = LexerMode::override_for_test(LexerMode::New);
840            assert_eq!(LexerMode::resolve(), LexerMode::New);
841        }
842        let _guard = LexerMode::override_for_test(LexerMode::Old);
843        assert_eq!(LexerMode::resolve(), LexerMode::Old);
844    }
845
846    #[test]
847    fn runtime_flags_default_to_unset() {
848        reset_runtime_lexer_flags();
849        assert_eq!(
850            crate::kcl_runtime_flags(),
851            KclRuntimeFlags {
852                use_new_lexer_parser: RuntimeFlag::Unset,
853                ..Default::default()
854            }
855        );
856    }
857
858    #[test]
859    fn runtime_flag_on_selects_new_lexer() {
860        reset_runtime_lexer_flags();
861        set_runtime_lexer_flag(RuntimeFlag::On);
862        assert_eq!(LexerMode::resolve(), LexerMode::New);
863    }
864
865    #[test]
866    fn runtime_flag_off_selects_old_lexer() {
867        reset_runtime_lexer_flags();
868        set_runtime_lexer_flag(RuntimeFlag::Off);
869        assert_eq!(LexerMode::resolve(), LexerMode::Old);
870    }
871
872    #[test]
873    fn runtime_flag_takes_priority_over_test_override_and_env() {
874        assert_eq!(
875            LexerMode::resolve_from_sources(RuntimeFlag::Off, Some(LexerMode::New), Some("new")),
876            LexerMode::Old
877        );
878        assert_eq!(
879            LexerMode::resolve_from_sources(RuntimeFlag::On, Some(LexerMode::Old), Some("old")),
880            LexerMode::New
881        );
882    }
883
884    #[test]
885    fn unset_runtime_flag_allows_env_to_select_lexer() {
886        assert_eq!(
887            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, Some("new")),
888            LexerMode::New
889        );
890        assert_eq!(
891            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, Some("old")),
892            LexerMode::Old
893        );
894    }
895
896    #[test]
897    fn unset_runtime_flag_and_missing_env_selects_default_lexer() {
898        assert_eq!(
899            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, None),
900            LexerMode::DEFAULT
901        );
902    }
903
904    #[test]
905    fn test_override_takes_priority_over_env() {
906        assert_eq!(
907            LexerMode::resolve_from_sources(RuntimeFlag::Unset, Some(LexerMode::Old), Some("new")),
908            LexerMode::Old
909        );
910        assert_eq!(
911            LexerMode::resolve_from_sources(RuntimeFlag::Unset, Some(LexerMode::New), Some("old")),
912            LexerMode::New
913        );
914    }
915
916    /// Exercises the `New` arm of `lex` in default CI: no `KCL_LEXER` env var is
917    /// set; the new lexer is selected via the process-global test override (which
918    /// is race-free under nextest's process-per-test isolation).
919    ///
920    /// The unterminated-string assertion is deliberately a *distinguishing* one:
921    /// the new lexer folds the recovery token into the message "unterminated
922    /// string literal", whereas the old lexer reports `found unknown token '"'`.
923    /// Asserting the new-lexer-only message proves `lex` took the `New` arm --
924    /// not merely that some lexer ran.
925    #[test]
926    fn lex_dispatches_to_new_lexer() {
927        reset_runtime_lexer_flags();
928        let _guard = LexerMode::override_for_test(LexerMode::New);
929        assert_eq!(LexerMode::resolve(), LexerMode::New);
930
931        let module_id = ModuleId::default();
932
933        // Valid input flows through the New arm and yields a token stream.
934        let tokens = lex("x = 1", module_id).expect("new lexer should tokenize valid input");
935        assert!(!tokens.is_empty(), "expected a non-empty token stream");
936
937        // Unterminated string: the new-lexer-only message (see doc comment).
938        let err = lex("\"abc", module_id).expect_err("unterminated string is a lexical error");
939        assert_eq!(err.error_type(), "lexical");
940        assert_eq!(err.message(), "unterminated string literal");
941    }
942}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
