kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::env;
5use std::fmt;
6use std::iter::Enumerate;
7use std::num::NonZeroUsize;
8use std::str::FromStr;
9
10use anyhow::Result;
11use kcl_error::KclErrorDetails;
12use parse_display::Display;
13use serde::Deserialize;
14use serde::Serialize;
15use tower_lsp::lsp_types::SemanticTokenType;
16use winnow::stream::ContainsToken;
17use winnow::stream::Stream;
18use winnow::{self};
19
20use crate::CompilationIssue;
21use crate::ModuleId;
22use crate::RuntimeFlag;
23use crate::SourceRange;
24use crate::errors::KclError;
25use crate::kcl_runtime_flags;
26use crate::parsing::ast::types::ItemVisibility;
27use crate::parsing::ast::types::VariableKind;
28use crate::runtime_flags::RuntimeFlagResolve;
29use crate::runtime_flags::resolve_from_sources;
30
31mod tokeniser;
32
33pub(crate) mod adapter;
34
35#[cfg(test)]
36mod compat_tests;
37
38#[cfg(test)]
39mod error_matrix_tests;
40
41pub(crate) use tokeniser::RESERVED_SKETCH_BLOCK_WORDS;
42pub(crate) use tokeniser::RESERVED_WORDS;
43
44// Note the ordering, it's important that `m` comes after `mm` and `cm`.
45pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
46
47#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
48#[repr(u32)]
49pub enum NumericSuffix {
50    None,
51    Count,
52    Length,
53    Angle,
54    Mm,
55    Cm,
56    M,
57    Inch,
58    Ft,
59    Yd,
60    Deg,
61    Rad,
62    Unknown,
63}
64
65impl NumericSuffix {
66    #[allow(dead_code)]
67    pub fn is_none(self) -> bool {
68        self == Self::None
69    }
70
71    pub fn is_some(self) -> bool {
72        self != Self::None
73    }
74
75    pub fn digestable_id(&self) -> &[u8] {
76        match self {
77            NumericSuffix::None => &[],
78            NumericSuffix::Count => b"_",
79            NumericSuffix::Unknown => b"?",
80            NumericSuffix::Length => b"Length",
81            NumericSuffix::Angle => b"Angle",
82            NumericSuffix::Mm => b"mm",
83            NumericSuffix::Cm => b"cm",
84            NumericSuffix::M => b"m",
85            NumericSuffix::Inch => b"in",
86            NumericSuffix::Ft => b"ft",
87            NumericSuffix::Yd => b"yd",
88            NumericSuffix::Deg => b"deg",
89            NumericSuffix::Rad => b"rad",
90        }
91    }
92}
93
94impl FromStr for NumericSuffix {
95    type Err = CompilationIssue;
96
97    fn from_str(s: &str) -> Result<Self, Self::Err> {
98        match s {
99            "_" | "Count" => Ok(NumericSuffix::Count),
100            "Length" => Ok(NumericSuffix::Length),
101            "Angle" => Ok(NumericSuffix::Angle),
102            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
103            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
104            "m" | "meters" => Ok(NumericSuffix::M),
105            "inch" | "in" => Ok(NumericSuffix::Inch),
106            "ft" | "feet" => Ok(NumericSuffix::Ft),
107            "yd" | "yards" => Ok(NumericSuffix::Yd),
108            "deg" | "degrees" => Ok(NumericSuffix::Deg),
109            "rad" | "radians" => Ok(NumericSuffix::Rad),
110            "?" => Ok(NumericSuffix::Unknown),
111            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
112        }
113    }
114}
115
116impl fmt::Display for NumericSuffix {
117    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118        match self {
119            NumericSuffix::None => Ok(()),
120            NumericSuffix::Count => write!(f, "_"),
121            NumericSuffix::Unknown => write!(f, "_?"),
122            NumericSuffix::Length => write!(f, "Length"),
123            NumericSuffix::Angle => write!(f, "Angle"),
124            NumericSuffix::Mm => write!(f, "mm"),
125            NumericSuffix::Cm => write!(f, "cm"),
126            NumericSuffix::M => write!(f, "m"),
127            NumericSuffix::Inch => write!(f, "in"),
128            NumericSuffix::Ft => write!(f, "ft"),
129            NumericSuffix::Yd => write!(f, "yd"),
130            NumericSuffix::Deg => write!(f, "deg"),
131            NumericSuffix::Rad => write!(f, "rad"),
132        }
133    }
134}
135
136#[derive(Clone, Debug, PartialEq)]
137pub(crate) struct TokenStream {
138    tokens: Vec<Token>,
139}
140
141impl TokenStream {
142    fn new(tokens: Vec<Token>) -> Self {
143        Self { tokens }
144    }
145
146    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
147        let tokens = std::mem::take(&mut self.tokens);
148        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
149            .into_iter()
150            .partition(|token| token.token_type != TokenType::Unknown);
151        self.tokens = tokens;
152        unknown_tokens
153    }
154
155    pub fn iter(&self) -> impl Iterator<Item = &Token> {
156        self.tokens.iter()
157    }
158
159    pub fn is_empty(&self) -> bool {
160        self.tokens.is_empty()
161    }
162
163    pub fn as_slice(&self) -> TokenSlice<'_> {
164        TokenSlice::from(self)
165    }
166}
167
168impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
169    fn from(stream: &'a TokenStream) -> Self {
170        TokenSlice {
171            start: 0,
172            end: stream.tokens.len(),
173            stream,
174        }
175    }
176}
177
178impl IntoIterator for TokenStream {
179    type Item = Token;
180
181    type IntoIter = std::vec::IntoIter<Token>;
182
183    fn into_iter(self) -> Self::IntoIter {
184        self.tokens.into_iter()
185    }
186}
187
188#[derive(Debug, Clone)]
189pub(crate) struct TokenSlice<'a> {
190    stream: &'a TokenStream,
191    /// Current position of the leading Token in the stream
192    start: usize,
193    /// The number of total Tokens in the stream
194    end: usize,
195}
196
197impl<'a> std::ops::Deref for TokenSlice<'a> {
198    type Target = [Token];
199
200    fn deref(&self) -> &Self::Target {
201        &self.stream.tokens[self.start..self.end]
202    }
203}
204
205impl<'a> TokenSlice<'a> {
206    pub fn token(&self, i: usize) -> &Token {
207        &self.stream.tokens[i + self.start]
208    }
209
210    pub fn iter(&self) -> impl Iterator<Item = &Token> {
211        (**self).iter()
212    }
213
214    pub fn without_ends(&self) -> Self {
215        Self {
216            start: self.start + 1,
217            end: self.end - 1,
218            stream: self.stream,
219        }
220    }
221
222    pub fn as_source_range(&self) -> SourceRange {
223        let stream_len = self.stream.tokens.len();
224        let first_token = if stream_len == self.start {
225            &self.stream.tokens[self.start - 1]
226        } else {
227            self.token(0)
228        };
229        let last_token = if stream_len == self.end {
230            &self.stream.tokens[stream_len - 1]
231        } else {
232            self.token(self.end - self.start)
233        };
234        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
235    }
236}
237
238impl<'a> IntoIterator for TokenSlice<'a> {
239    type Item = &'a Token;
240
241    type IntoIter = std::slice::Iter<'a, Token>;
242
243    fn into_iter(self) -> Self::IntoIter {
244        self.stream.tokens[self.start..self.end].iter()
245    }
246}
247
248impl<'a> Stream for TokenSlice<'a> {
249    type Token = Token;
250    type Slice = Self;
251    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
252    type Checkpoint = Checkpoint;
253
254    fn iter_offsets(&self) -> Self::IterOffsets {
255        #[allow(clippy::unnecessary_to_owned)]
256        self.to_vec().into_iter().enumerate()
257    }
258
259    fn eof_offset(&self) -> usize {
260        self.len()
261    }
262
263    fn next_token(&mut self) -> Option<Self::Token> {
264        let token = self.first()?.clone();
265        self.start += 1;
266        Some(token)
267    }
268
269    /// Split off the next token from the input
270    fn peek_token(&self) -> Option<Self::Token> {
271        Some(self.first()?.clone())
272    }
273
274    fn offset_for<P>(&self, predicate: P) -> Option<usize>
275    where
276        P: Fn(Self::Token) -> bool,
277    {
278        self.iter().position(|b| predicate(b.clone()))
279    }
280
281    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
282        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
283            Err(winnow::error::Needed::Size(needed))
284        } else {
285            Ok(tokens)
286        }
287    }
288
289    fn next_slice(&mut self, offset: usize) -> Self::Slice {
290        assert!(self.start + offset <= self.end);
291
292        let next = TokenSlice {
293            stream: self.stream,
294            start: self.start,
295            end: self.start + offset,
296        };
297        self.start += offset;
298        next
299    }
300
301    /// Split off a slice of tokens from the input
302    fn peek_slice(&self, offset: usize) -> Self::Slice {
303        assert!(self.start + offset <= self.end);
304
305        TokenSlice {
306            stream: self.stream,
307            start: self.start,
308            end: self.start + offset,
309        }
310    }
311
312    fn checkpoint(&self) -> Self::Checkpoint {
313        Checkpoint(self.start, self.end)
314    }
315
316    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
317        self.start = checkpoint.0;
318        self.end = checkpoint.1;
319    }
320
321    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
322        write!(f, "{self:?}")
323    }
324}
325
326impl<'a> winnow::stream::Offset for TokenSlice<'a> {
327    fn offset_from(&self, start: &Self) -> usize {
328        self.start - start.start
329    }
330}
331
332impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
333    fn offset_from(&self, start: &Checkpoint) -> usize {
334        self.start - start.0
335    }
336}
337
338impl winnow::stream::Offset for Checkpoint {
339    fn offset_from(&self, start: &Self) -> usize {
340        self.0 - start.0
341    }
342}
343
344impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
345    type PartialState = ();
346
347    fn complete(&mut self) -> Self::PartialState {}
348
349    fn restore_partial(&mut self, _: Self::PartialState) {}
350
351    fn is_partial_supported() -> bool {
352        false
353    }
354}
355
356impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
357    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
358        self.iter()
359            .enumerate()
360            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
361    }
362}
363
364#[derive(Clone, Debug)]
365pub struct Checkpoint(usize, usize);
366
367/// The types of tokens.
368#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
369#[display(style = "camelCase")]
370pub enum TokenType {
371    /// A number.
372    Number,
373    /// A word.
374    Word,
375    /// An operator.
376    Operator,
377    /// A string.
378    String,
379    /// A keyword.
380    Keyword,
381    /// A type.
382    Type,
383    /// A brace.
384    Brace,
385    /// A hash.
386    Hash,
387    /// A bang.
388    Bang,
389    /// A dollar sign.
390    Dollar,
391    /// Whitespace.
392    Whitespace,
393    /// A comma.
394    Comma,
395    /// A colon.
396    Colon,
397    /// A double colon: `::`
398    DoubleColon,
399    /// A period.
400    Period,
401    /// A double period: `..`.
402    DoublePeriod,
403    /// A double period and a less than: `..<`.
404    DoublePeriodLessThan,
405    /// A line comment.
406    LineComment,
407    /// A block comment.
408    BlockComment,
409    /// A function name.
410    Function,
411    /// Unknown lexemes.
412    Unknown,
413    /// The ? symbol, used for optional values.
414    QuestionMark,
415    /// The @ symbol.
416    At,
417    /// `;`
418    SemiColon,
419}
420
421/// Most KCL tokens correspond to LSP semantic tokens (but not all).
422impl TryFrom<TokenType> for SemanticTokenType {
423    type Error = anyhow::Error;
424    fn try_from(token_type: TokenType) -> Result<Self> {
425        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
426        // in the LSP implementation.
427        Ok(match token_type {
428            TokenType::Number => Self::NUMBER,
429            TokenType::Word => Self::VARIABLE,
430            TokenType::Keyword => Self::KEYWORD,
431            TokenType::Type => Self::TYPE,
432            TokenType::Operator => Self::OPERATOR,
433            TokenType::QuestionMark => Self::OPERATOR,
434            TokenType::String => Self::STRING,
435            TokenType::Bang => Self::OPERATOR,
436            TokenType::LineComment => Self::COMMENT,
437            TokenType::BlockComment => Self::COMMENT,
438            TokenType::Function => Self::FUNCTION,
439            TokenType::Whitespace
440            | TokenType::Brace
441            | TokenType::Comma
442            | TokenType::Colon
443            | TokenType::DoubleColon
444            | TokenType::Period
445            | TokenType::DoublePeriod
446            | TokenType::DoublePeriodLessThan
447            | TokenType::Hash
448            | TokenType::Dollar
449            | TokenType::At
450            | TokenType::SemiColon
451            | TokenType::Unknown => {
452                anyhow::bail!("unsupported token type: {:?}", token_type)
453            }
454        })
455    }
456}
457
458impl TokenType {
459    pub fn is_whitespace(&self) -> bool {
460        matches!(self, Self::Whitespace)
461    }
462
463    pub fn is_comment(&self) -> bool {
464        matches!(self, Self::LineComment | Self::BlockComment)
465    }
466}
467
468#[derive(Debug, PartialEq, Eq, Clone)]
469pub struct Token {
470    pub token_type: TokenType,
471    /// Offset in the source code where this token begins.
472    pub start: usize,
473    /// Offset in the source code where this token ends.
474    pub end: usize,
475    pub(super) module_id: ModuleId,
476    pub(super) value: String,
477}
478
479impl ContainsToken<Token> for (TokenType, &str) {
480    fn contains_token(&self, token: Token) -> bool {
481        self.0 == token.token_type && self.1 == token.value
482    }
483}
484
485impl ContainsToken<Token> for TokenType {
486    fn contains_token(&self, token: Token) -> bool {
487        *self == token.token_type
488    }
489}
490
491impl Token {
492    pub fn from_range(
493        range: std::ops::Range<usize>,
494        module_id: ModuleId,
495        token_type: TokenType,
496        value: String,
497    ) -> Self {
498        Self {
499            start: range.start,
500            end: range.end,
501            module_id,
502            value,
503            token_type,
504        }
505    }
506    pub fn is_code_token(&self) -> bool {
507        !matches!(
508            self.token_type,
509            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
510        )
511    }
512
513    pub fn as_source_range(&self) -> SourceRange {
514        SourceRange::new(self.start, self.end, self.module_id)
515    }
516
517    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
518        vec![self.as_source_range()]
519    }
520
521    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
522        if !matches!(self.token_type, TokenType::Keyword) {
523            return None;
524        }
525        match self.value.as_str() {
526            "export" => Some(ItemVisibility::Export),
527            _ => None,
528        }
529    }
530
531    pub fn numeric_value(&self) -> Option<f64> {
532        if self.token_type != TokenType::Number {
533            return None;
534        }
535        let value = &self.value;
536        let value = value
537            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
538            .map(|(s, _)| s)
539            .unwrap_or(value);
540        value.parse().ok()
541    }
542
543    pub fn uint_value(&self) -> Option<u32> {
544        if self.token_type != TokenType::Number {
545            return None;
546        }
547        let value = &self.value;
548        let value = value
549            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
550            .map(|(s, _)| s)
551            .unwrap_or(value);
552        value.parse().ok()
553    }
554
555    pub fn numeric_suffix(&self) -> NumericSuffix {
556        if self.token_type != TokenType::Number {
557            return NumericSuffix::None;
558        }
559
560        if self.value.ends_with('_') {
561            return NumericSuffix::Count;
562        }
563
564        for suffix in NUM_SUFFIXES {
565            if self.value.ends_with(suffix) {
566                return suffix.parse().unwrap();
567            }
568        }
569
570        NumericSuffix::None
571    }
572
573    /// Is this token the beginning of a variable/function declaration?
574    /// If so, what kind?
575    /// If not, returns None.
576    pub fn declaration_keyword(&self) -> Option<VariableKind> {
577        if !matches!(self.token_type, TokenType::Keyword) {
578            return None;
579        }
580        Some(match self.value.as_str() {
581            "fn" => VariableKind::Fn,
582            "var" | "let" | "const" => VariableKind::Const,
583            _ => return None,
584        })
585    }
586}
587
588impl From<Token> for SourceRange {
589    fn from(token: Token) -> Self {
590        Self::new(token.start, token.end, token.module_id)
591    }
592}
593
594impl From<&Token> for SourceRange {
595    fn from(token: &Token) -> Self {
596        Self::new(token.start, token.end, token.module_id)
597    }
598}
599
600/// Environment variable selecting which lexer implementation [`lex`] uses.
601pub(crate) const KCL_LEXER_ENV_VAR: &str = "KCL_LEXER";
602
603/// Which lexer implementation [`lex`] uses: the old winnow `tokeniser` (`Old`) or
604/// the new `kcl-syntax` logos lexer (`New`). Selected at runtime via the
605/// `KCL_LEXER` environment variable, so a process can pick either lexer without a
606/// rebuild.
607///
608/// Precedence: runtime flags > test override > `KCL_LEXER` >
609/// [`LexerMode::DEFAULT`].
610#[derive(Debug, Clone, Copy, PartialEq, Eq)]
611pub(crate) enum LexerMode {
612    Old,
613    New,
614}
615
616impl RuntimeFlagResolve for LexerMode {
617    fn on() -> Self {
618        Self::New
619    }
620
621    fn off() -> Self {
622        Self::Old
623    }
624
625    fn resolve_default() -> Self {
626        Self::DEFAULT
627    }
628
629    fn parse_env_var(value: &str) -> Self {
630        Self::parse(value)
631    }
632}
633
634impl LexerMode {
635    /// The mode used when `KCL_LEXER` is unset.
636    const DEFAULT: Self = Self::Old;
637
638    /// Resolve the active lexer mode (see precedence on [`LexerMode`]).
639    pub(crate) fn resolve() -> Self {
640        let env_value = match env::var(KCL_LEXER_ENV_VAR) {
641            Ok(value) => Some(value),
642            Err(env::VarError::NotPresent) => None,
643            Err(env::VarError::NotUnicode(value)) => {
644                // Invalid-unicode env var: warn and fall back rather than crash.
645                Self::warn_once(|| {
646                    format!(
647                        "{KCL_LEXER_ENV_VAR} must be valid unicode; got `{}`. Defaulting to `old`.",
648                        value.to_string_lossy()
649                    )
650                });
651                None
652            }
653        };
654
655        Self::resolve_from_sources(
656            kcl_runtime_flags().use_new_lexer_parser,
657            Self::test_override_for_resolve(),
658            env_value.as_deref(),
659        )
660    }
661
662    fn resolve_from_sources(runtime_flag: RuntimeFlag, test_override: Option<Self>, env_value: Option<&str>) -> Self {
663        resolve_from_sources(runtime_flag, test_override, env_value)
664    }
665
666    #[cfg(test)]
667    fn test_override_for_resolve() -> Option<Self> {
668        Self::test_override()
669    }
670
671    #[cfg(not(test))]
672    fn test_override_for_resolve() -> Option<Self> {
673        None
674    }
675
676    fn parse(value: &str) -> Self {
677        let value = value.trim();
678        if value.eq_ignore_ascii_case("old") {
679            return Self::Old;
680        }
681        if value.eq_ignore_ascii_case("new") {
682            return Self::New;
683        }
684
685        // A mistyped `KCL_LEXER` should not crash the process: warn and fall back
686        // to the old lexer (the conservative choice for a misconfiguration).
687        Self::warn_once(|| {
688            format!("Unsupported {KCL_LEXER_ENV_VAR} value `{value}`; expected `old` or `new`. Defaulting to `old`.")
689        });
690        Self::Old
691    }
692
693    /// Emit a one-time configuration warning through `crate::log` (gated on
694    /// `ZOO_LOG`). `resolve`/`parse` run on every `lex`, so a misconfigured
695    /// `KCL_LEXER` must not warn -- or allocate the message -- on every call. One
696    /// guard suffices: only one kind of misconfiguration can occur per process,
697    /// since the env var holds a single value.
698    fn warn_once(make_message: impl FnOnce() -> String) {
699        static WARNED: std::sync::Once = std::sync::Once::new();
700        WARNED.call_once(|| crate::log::log(make_message()));
701    }
702
703    #[cfg(test)]
704    fn test_override_value(self) -> u8 {
705        match self {
706            Self::Old => 1,
707            Self::New => 2,
708        }
709    }
710
711    #[cfg(test)]
712    fn test_override() -> Option<Self> {
713        match TEST_LEXER_MODE_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
714            1 => Some(Self::Old),
715            2 => Some(Self::New),
716            _ => None,
717        }
718    }
719
720    /// Override the lexer mode for the lifetime of the returned guard.
721    ///
722    /// This uses a process-global atomic, so it is only race-free under test
723    /// runners that isolate tests in separate processes (e.g. `cargo nextest`).
724    /// Under in-process parallel `cargo test`, prefer driving the lexer with an
725    /// explicit mode; reserve this guard for dispatch/integration tests.
726    #[cfg(test)]
727    pub(crate) fn override_for_test(mode: Self) -> LexerModeOverrideGuard {
728        let previous = TEST_LEXER_MODE_OVERRIDE.swap(mode.test_override_value(), std::sync::atomic::Ordering::SeqCst);
729        LexerModeOverrideGuard { previous }
730    }
731}
732
733#[cfg(test)]
734static TEST_LEXER_MODE_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
735
736#[cfg(test)]
737pub(crate) struct LexerModeOverrideGuard {
738    previous: u8,
739}
740
741#[cfg(test)]
742impl Drop for LexerModeOverrideGuard {
743    fn drop(&mut self) {
744        TEST_LEXER_MODE_OVERRIDE.store(self.previous, std::sync::atomic::Ordering::SeqCst);
745    }
746}
747
748// `lex` dispatches on the runtime `LexerMode`. `Old` runs the winnow
749// `tokeniser`; `New` runs the `kcl-syntax` adapter and folds any fatal lexical
750// diagnostics into a single lexical `KclError`, preserving the public `Result`
751// contract. (The LSP consumes the richer `LexResult` directly so it can keep
752// tokens for highlighting while reporting diagnostics.)
753pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
754    match LexerMode::resolve() {
755        LexerMode::Old => lex_legacy(s, module_id),
756        LexerMode::New => {
757            let result = adapter::lex_with_diagnostics(s, module_id);
758            match result.to_lexical_error() {
759                Some(err) => Err(err),
760                None => Ok(result.tokens),
761            }
762        }
763    }
764}
765
766fn lex_legacy(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
767    tokeniser::lex(s, module_id).map_err(|err| {
768        let (input, offset): (Vec<char>, usize) = (err.input().chars().collect(), err.offset());
769        let module_id = err.input().state.module_id;
770
771        if offset >= input.len() {
772            // From the winnow docs:
773            //
774            // This is an offset, not an index, and may point to
775            // the end of input (input.len()) on eof errors.
776
777            return KclError::new_lexical(KclErrorDetails::new(
778                "unexpected EOF while parsing".to_owned(),
779                vec![SourceRange::new(offset, offset, module_id)],
780            ));
781        }
782
783        // TODO: Add the Winnow tokenizer context to the error.
784        // See https://github.com/KittyCAD/modeling-app/issues/784
785        let bad_token = &input[offset];
786        // TODO: Add the Winnow parser context to the error.
787        // See https://github.com/KittyCAD/modeling-app/issues/784
788        KclError::new_lexical(KclErrorDetails::new(
789            format!("found unknown token '{bad_token}'"),
790            vec![SourceRange::new(offset, offset + 1, module_id)],
791        ))
792    })
793}
794
795#[cfg(test)]
796mod lexer_mode_tests {
797    use super::LexerMode;
798    use super::lex;
799    use crate::KclRuntimeFlags;
800    use crate::ModuleId;
801    use crate::RuntimeFlag;
802
803    fn set_runtime_lexer_flag(flag: RuntimeFlag) {
804        crate::set_kcl_runtime_flags(KclRuntimeFlags {
805            use_new_lexer_parser: flag,
806            ..Default::default()
807        });
808    }
809
810    fn reset_runtime_lexer_flags() {
811        crate::set_kcl_runtime_flags(KclRuntimeFlags::DEFAULT);
812    }
813
814    #[test]
815    fn default_mode_is_old() {
816        reset_runtime_lexer_flags();
817        assert_eq!(LexerMode::DEFAULT, LexerMode::Old);
818    }
819
820    #[test]
821    fn parse_accepts_known_values_case_insensitively() {
822        assert_eq!(LexerMode::parse("old"), LexerMode::Old);
823        assert_eq!(LexerMode::parse("  NEW  "), LexerMode::New);
824    }
825
826    #[test]
827    fn parse_falls_back_to_old_on_unknown_value() {
828        // An unknown value warns and defaults to the old lexer instead of panicking.
829        assert_eq!(LexerMode::parse("rowan"), LexerMode::Old);
830    }
831
832    #[test]
833    fn override_guard_sets_and_restores_mode() {
834        reset_runtime_lexer_flags();
835        // Reserved for dispatch/integration tests; relies on the process-global
836        // atomic, which is race-free under nextest's process isolation.
837        {
838            let _guard = LexerMode::override_for_test(LexerMode::New);
839            assert_eq!(LexerMode::resolve(), LexerMode::New);
840        }
841        let _guard = LexerMode::override_for_test(LexerMode::Old);
842        assert_eq!(LexerMode::resolve(), LexerMode::Old);
843    }
844
845    #[test]
846    fn runtime_flags_default_to_unset() {
847        reset_runtime_lexer_flags();
848        assert_eq!(
849            crate::kcl_runtime_flags(),
850            KclRuntimeFlags {
851                use_new_lexer_parser: RuntimeFlag::Unset,
852                ..Default::default()
853            }
854        );
855    }
856
857    #[test]
858    fn runtime_flag_on_selects_new_lexer() {
859        reset_runtime_lexer_flags();
860        set_runtime_lexer_flag(RuntimeFlag::On);
861        assert_eq!(LexerMode::resolve(), LexerMode::New);
862    }
863
864    #[test]
865    fn runtime_flag_off_selects_old_lexer() {
866        reset_runtime_lexer_flags();
867        set_runtime_lexer_flag(RuntimeFlag::Off);
868        assert_eq!(LexerMode::resolve(), LexerMode::Old);
869    }
870
871    #[test]
872    fn runtime_flag_takes_priority_over_test_override_and_env() {
873        assert_eq!(
874            LexerMode::resolve_from_sources(RuntimeFlag::Off, Some(LexerMode::New), Some("new")),
875            LexerMode::Old
876        );
877        assert_eq!(
878            LexerMode::resolve_from_sources(RuntimeFlag::On, Some(LexerMode::Old), Some("old")),
879            LexerMode::New
880        );
881    }
882
883    #[test]
884    fn unset_runtime_flag_allows_env_to_select_lexer() {
885        assert_eq!(
886            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, Some("new")),
887            LexerMode::New
888        );
889        assert_eq!(
890            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, Some("old")),
891            LexerMode::Old
892        );
893    }
894
895    #[test]
896    fn unset_runtime_flag_and_missing_env_selects_default_lexer() {
897        assert_eq!(
898            LexerMode::resolve_from_sources(RuntimeFlag::Unset, None, None),
899            LexerMode::DEFAULT
900        );
901    }
902
903    #[test]
904    fn test_override_takes_priority_over_env() {
905        assert_eq!(
906            LexerMode::resolve_from_sources(RuntimeFlag::Unset, Some(LexerMode::Old), Some("new")),
907            LexerMode::Old
908        );
909        assert_eq!(
910            LexerMode::resolve_from_sources(RuntimeFlag::Unset, Some(LexerMode::New), Some("old")),
911            LexerMode::New
912        );
913    }
914
915    /// Exercises the `New` arm of `lex` in default CI: no `KCL_LEXER` env var is
916    /// set; the new lexer is selected via the process-global test override (which
917    /// is race-free under nextest's process-per-test isolation).
918    ///
919    /// The unterminated-string assertion is deliberately a *distinguishing* one:
920    /// the new lexer folds the recovery token into the message "unterminated
921    /// string literal", whereas the old lexer reports `found unknown token '"'`.
922    /// Asserting the new-lexer-only message proves `lex` took the `New` arm --
923    /// not merely that some lexer ran.
924    #[test]
925    fn lex_dispatches_to_new_lexer() {
926        reset_runtime_lexer_flags();
927        let _guard = LexerMode::override_for_test(LexerMode::New);
928        assert_eq!(LexerMode::resolve(), LexerMode::New);
929
930        let module_id = ModuleId::default();
931
932        // Valid input flows through the New arm and yields a token stream.
933        let tokens = lex("x = 1", module_id).expect("new lexer should tokenize valid input");
934        assert!(!tokens.is_empty(), "expected a non-empty token stream");
935
936        // Unterminated string: the new-lexer-only message (see doc comment).
937        let err = lex("\"abc", module_id).expect_err("unterminated string is a lexical error");
938        assert_eq!(err.error_type(), "lexical");
939        assert_eq!(err.message(), "unterminated string literal");
940    }
941}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
