kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::env;
5use std::fmt;
6use std::iter::Enumerate;
7use std::num::NonZeroUsize;
8use std::str::FromStr;
9
10use anyhow::Result;
11use kcl_error::KclErrorDetails;
12use parse_display::Display;
13use serde::Deserialize;
14use serde::Serialize;
15use tower_lsp::lsp_types::SemanticTokenType;
16use winnow::stream::ContainsToken;
17use winnow::stream::Stream;
18use winnow::{self};
19
20use crate::CompilationIssue;
21use crate::ModuleId;
22use crate::SourceRange;
23use crate::errors::KclError;
24use crate::parsing::ast::types::ItemVisibility;
25use crate::parsing::ast::types::VariableKind;
26
27mod tokeniser;
28
29pub(crate) mod adapter;
30
31#[cfg(test)]
32mod compat_tests;
33
34#[cfg(test)]
35mod error_matrix_tests;
36
37pub(crate) use tokeniser::RESERVED_SKETCH_BLOCK_WORDS;
38pub(crate) use tokeniser::RESERVED_WORDS;
39
40// Note the ordering, it's important that `m` comes after `mm` and `cm`.
41pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
42
43#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
44#[repr(u32)]
45pub enum NumericSuffix {
46    None,
47    Count,
48    Length,
49    Angle,
50    Mm,
51    Cm,
52    M,
53    Inch,
54    Ft,
55    Yd,
56    Deg,
57    Rad,
58    Unknown,
59}
60
61impl NumericSuffix {
62    #[allow(dead_code)]
63    pub fn is_none(self) -> bool {
64        self == Self::None
65    }
66
67    pub fn is_some(self) -> bool {
68        self != Self::None
69    }
70
71    pub fn digestable_id(&self) -> &[u8] {
72        match self {
73            NumericSuffix::None => &[],
74            NumericSuffix::Count => b"_",
75            NumericSuffix::Unknown => b"?",
76            NumericSuffix::Length => b"Length",
77            NumericSuffix::Angle => b"Angle",
78            NumericSuffix::Mm => b"mm",
79            NumericSuffix::Cm => b"cm",
80            NumericSuffix::M => b"m",
81            NumericSuffix::Inch => b"in",
82            NumericSuffix::Ft => b"ft",
83            NumericSuffix::Yd => b"yd",
84            NumericSuffix::Deg => b"deg",
85            NumericSuffix::Rad => b"rad",
86        }
87    }
88}
89
90impl FromStr for NumericSuffix {
91    type Err = CompilationIssue;
92
93    fn from_str(s: &str) -> Result<Self, Self::Err> {
94        match s {
95            "_" | "Count" => Ok(NumericSuffix::Count),
96            "Length" => Ok(NumericSuffix::Length),
97            "Angle" => Ok(NumericSuffix::Angle),
98            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
99            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
100            "m" | "meters" => Ok(NumericSuffix::M),
101            "inch" | "in" => Ok(NumericSuffix::Inch),
102            "ft" | "feet" => Ok(NumericSuffix::Ft),
103            "yd" | "yards" => Ok(NumericSuffix::Yd),
104            "deg" | "degrees" => Ok(NumericSuffix::Deg),
105            "rad" | "radians" => Ok(NumericSuffix::Rad),
106            "?" => Ok(NumericSuffix::Unknown),
107            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
108        }
109    }
110}
111
112impl fmt::Display for NumericSuffix {
113    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
114        match self {
115            NumericSuffix::None => Ok(()),
116            NumericSuffix::Count => write!(f, "_"),
117            NumericSuffix::Unknown => write!(f, "_?"),
118            NumericSuffix::Length => write!(f, "Length"),
119            NumericSuffix::Angle => write!(f, "Angle"),
120            NumericSuffix::Mm => write!(f, "mm"),
121            NumericSuffix::Cm => write!(f, "cm"),
122            NumericSuffix::M => write!(f, "m"),
123            NumericSuffix::Inch => write!(f, "in"),
124            NumericSuffix::Ft => write!(f, "ft"),
125            NumericSuffix::Yd => write!(f, "yd"),
126            NumericSuffix::Deg => write!(f, "deg"),
127            NumericSuffix::Rad => write!(f, "rad"),
128        }
129    }
130}
131
132#[derive(Clone, Debug, PartialEq)]
133pub(crate) struct TokenStream {
134    tokens: Vec<Token>,
135}
136
137impl TokenStream {
138    fn new(tokens: Vec<Token>) -> Self {
139        Self { tokens }
140    }
141
142    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
143        let tokens = std::mem::take(&mut self.tokens);
144        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
145            .into_iter()
146            .partition(|token| token.token_type != TokenType::Unknown);
147        self.tokens = tokens;
148        unknown_tokens
149    }
150
151    pub fn iter(&self) -> impl Iterator<Item = &Token> {
152        self.tokens.iter()
153    }
154
155    pub fn is_empty(&self) -> bool {
156        self.tokens.is_empty()
157    }
158
159    pub fn as_slice(&self) -> TokenSlice<'_> {
160        TokenSlice::from(self)
161    }
162}
163
164impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
165    fn from(stream: &'a TokenStream) -> Self {
166        TokenSlice {
167            start: 0,
168            end: stream.tokens.len(),
169            stream,
170        }
171    }
172}
173
174impl IntoIterator for TokenStream {
175    type Item = Token;
176
177    type IntoIter = std::vec::IntoIter<Token>;
178
179    fn into_iter(self) -> Self::IntoIter {
180        self.tokens.into_iter()
181    }
182}
183
184#[derive(Debug, Clone)]
185pub(crate) struct TokenSlice<'a> {
186    stream: &'a TokenStream,
187    /// Current position of the leading Token in the stream
188    start: usize,
189    /// The number of total Tokens in the stream
190    end: usize,
191}
192
193impl<'a> std::ops::Deref for TokenSlice<'a> {
194    type Target = [Token];
195
196    fn deref(&self) -> &Self::Target {
197        &self.stream.tokens[self.start..self.end]
198    }
199}
200
201impl<'a> TokenSlice<'a> {
202    pub fn token(&self, i: usize) -> &Token {
203        &self.stream.tokens[i + self.start]
204    }
205
206    pub fn iter(&self) -> impl Iterator<Item = &Token> {
207        (**self).iter()
208    }
209
210    pub fn without_ends(&self) -> Self {
211        Self {
212            start: self.start + 1,
213            end: self.end - 1,
214            stream: self.stream,
215        }
216    }
217
218    pub fn as_source_range(&self) -> SourceRange {
219        let stream_len = self.stream.tokens.len();
220        let first_token = if stream_len == self.start {
221            &self.stream.tokens[self.start - 1]
222        } else {
223            self.token(0)
224        };
225        let last_token = if stream_len == self.end {
226            &self.stream.tokens[stream_len - 1]
227        } else {
228            self.token(self.end - self.start)
229        };
230        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
231    }
232}
233
234impl<'a> IntoIterator for TokenSlice<'a> {
235    type Item = &'a Token;
236
237    type IntoIter = std::slice::Iter<'a, Token>;
238
239    fn into_iter(self) -> Self::IntoIter {
240        self.stream.tokens[self.start..self.end].iter()
241    }
242}
243
244impl<'a> Stream for TokenSlice<'a> {
245    type Token = Token;
246    type Slice = Self;
247    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
248    type Checkpoint = Checkpoint;
249
250    fn iter_offsets(&self) -> Self::IterOffsets {
251        #[allow(clippy::unnecessary_to_owned)]
252        self.to_vec().into_iter().enumerate()
253    }
254
255    fn eof_offset(&self) -> usize {
256        self.len()
257    }
258
259    fn next_token(&mut self) -> Option<Self::Token> {
260        let token = self.first()?.clone();
261        self.start += 1;
262        Some(token)
263    }
264
265    /// Split off the next token from the input
266    fn peek_token(&self) -> Option<Self::Token> {
267        Some(self.first()?.clone())
268    }
269
270    fn offset_for<P>(&self, predicate: P) -> Option<usize>
271    where
272        P: Fn(Self::Token) -> bool,
273    {
274        self.iter().position(|b| predicate(b.clone()))
275    }
276
277    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
278        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
279            Err(winnow::error::Needed::Size(needed))
280        } else {
281            Ok(tokens)
282        }
283    }
284
285    fn next_slice(&mut self, offset: usize) -> Self::Slice {
286        assert!(self.start + offset <= self.end);
287
288        let next = TokenSlice {
289            stream: self.stream,
290            start: self.start,
291            end: self.start + offset,
292        };
293        self.start += offset;
294        next
295    }
296
297    /// Split off a slice of tokens from the input
298    fn peek_slice(&self, offset: usize) -> Self::Slice {
299        assert!(self.start + offset <= self.end);
300
301        TokenSlice {
302            stream: self.stream,
303            start: self.start,
304            end: self.start + offset,
305        }
306    }
307
308    fn checkpoint(&self) -> Self::Checkpoint {
309        Checkpoint(self.start, self.end)
310    }
311
312    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
313        self.start = checkpoint.0;
314        self.end = checkpoint.1;
315    }
316
317    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
318        write!(f, "{self:?}")
319    }
320}
321
322impl<'a> winnow::stream::Offset for TokenSlice<'a> {
323    fn offset_from(&self, start: &Self) -> usize {
324        self.start - start.start
325    }
326}
327
328impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
329    fn offset_from(&self, start: &Checkpoint) -> usize {
330        self.start - start.0
331    }
332}
333
334impl winnow::stream::Offset for Checkpoint {
335    fn offset_from(&self, start: &Self) -> usize {
336        self.0 - start.0
337    }
338}
339
340impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
341    type PartialState = ();
342
343    fn complete(&mut self) -> Self::PartialState {}
344
345    fn restore_partial(&mut self, _: Self::PartialState) {}
346
347    fn is_partial_supported() -> bool {
348        false
349    }
350}
351
352impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
353    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
354        self.iter()
355            .enumerate()
356            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
357    }
358}
359
360#[derive(Clone, Debug)]
361pub struct Checkpoint(usize, usize);
362
363/// The types of tokens.
364#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
365#[display(style = "camelCase")]
366pub enum TokenType {
367    /// A number.
368    Number,
369    /// A word.
370    Word,
371    /// An operator.
372    Operator,
373    /// A string.
374    String,
375    /// A keyword.
376    Keyword,
377    /// A type.
378    Type,
379    /// A brace.
380    Brace,
381    /// A hash.
382    Hash,
383    /// A bang.
384    Bang,
385    /// A dollar sign.
386    Dollar,
387    /// Whitespace.
388    Whitespace,
389    /// A comma.
390    Comma,
391    /// A colon.
392    Colon,
393    /// A double colon: `::`
394    DoubleColon,
395    /// A period.
396    Period,
397    /// A double period: `..`.
398    DoublePeriod,
399    /// A double period and a less than: `..<`.
400    DoublePeriodLessThan,
401    /// A line comment.
402    LineComment,
403    /// A block comment.
404    BlockComment,
405    /// A function name.
406    Function,
407    /// Unknown lexemes.
408    Unknown,
409    /// The ? symbol, used for optional values.
410    QuestionMark,
411    /// The @ symbol.
412    At,
413    /// `;`
414    SemiColon,
415}
416
417/// Most KCL tokens correspond to LSP semantic tokens (but not all).
418impl TryFrom<TokenType> for SemanticTokenType {
419    type Error = anyhow::Error;
420    fn try_from(token_type: TokenType) -> Result<Self> {
421        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
422        // in the LSP implementation.
423        Ok(match token_type {
424            TokenType::Number => Self::NUMBER,
425            TokenType::Word => Self::VARIABLE,
426            TokenType::Keyword => Self::KEYWORD,
427            TokenType::Type => Self::TYPE,
428            TokenType::Operator => Self::OPERATOR,
429            TokenType::QuestionMark => Self::OPERATOR,
430            TokenType::String => Self::STRING,
431            TokenType::Bang => Self::OPERATOR,
432            TokenType::LineComment => Self::COMMENT,
433            TokenType::BlockComment => Self::COMMENT,
434            TokenType::Function => Self::FUNCTION,
435            TokenType::Whitespace
436            | TokenType::Brace
437            | TokenType::Comma
438            | TokenType::Colon
439            | TokenType::DoubleColon
440            | TokenType::Period
441            | TokenType::DoublePeriod
442            | TokenType::DoublePeriodLessThan
443            | TokenType::Hash
444            | TokenType::Dollar
445            | TokenType::At
446            | TokenType::SemiColon
447            | TokenType::Unknown => {
448                anyhow::bail!("unsupported token type: {:?}", token_type)
449            }
450        })
451    }
452}
453
454impl TokenType {
455    pub fn is_whitespace(&self) -> bool {
456        matches!(self, Self::Whitespace)
457    }
458
459    pub fn is_comment(&self) -> bool {
460        matches!(self, Self::LineComment | Self::BlockComment)
461    }
462}
463
464#[derive(Debug, PartialEq, Eq, Clone)]
465pub struct Token {
466    pub token_type: TokenType,
467    /// Offset in the source code where this token begins.
468    pub start: usize,
469    /// Offset in the source code where this token ends.
470    pub end: usize,
471    pub(super) module_id: ModuleId,
472    pub(super) value: String,
473}
474
475impl ContainsToken<Token> for (TokenType, &str) {
476    fn contains_token(&self, token: Token) -> bool {
477        self.0 == token.token_type && self.1 == token.value
478    }
479}
480
481impl ContainsToken<Token> for TokenType {
482    fn contains_token(&self, token: Token) -> bool {
483        *self == token.token_type
484    }
485}
486
487impl Token {
488    pub fn from_range(
489        range: std::ops::Range<usize>,
490        module_id: ModuleId,
491        token_type: TokenType,
492        value: String,
493    ) -> Self {
494        Self {
495            start: range.start,
496            end: range.end,
497            module_id,
498            value,
499            token_type,
500        }
501    }
502    pub fn is_code_token(&self) -> bool {
503        !matches!(
504            self.token_type,
505            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
506        )
507    }
508
509    pub fn as_source_range(&self) -> SourceRange {
510        SourceRange::new(self.start, self.end, self.module_id)
511    }
512
513    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
514        vec![self.as_source_range()]
515    }
516
517    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
518        if !matches!(self.token_type, TokenType::Keyword) {
519            return None;
520        }
521        match self.value.as_str() {
522            "export" => Some(ItemVisibility::Export),
523            _ => None,
524        }
525    }
526
527    pub fn numeric_value(&self) -> Option<f64> {
528        if self.token_type != TokenType::Number {
529            return None;
530        }
531        let value = &self.value;
532        let value = value
533            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
534            .map(|(s, _)| s)
535            .unwrap_or(value);
536        value.parse().ok()
537    }
538
539    pub fn uint_value(&self) -> Option<u32> {
540        if self.token_type != TokenType::Number {
541            return None;
542        }
543        let value = &self.value;
544        let value = value
545            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
546            .map(|(s, _)| s)
547            .unwrap_or(value);
548        value.parse().ok()
549    }
550
551    pub fn numeric_suffix(&self) -> NumericSuffix {
552        if self.token_type != TokenType::Number {
553            return NumericSuffix::None;
554        }
555
556        if self.value.ends_with('_') {
557            return NumericSuffix::Count;
558        }
559
560        for suffix in NUM_SUFFIXES {
561            if self.value.ends_with(suffix) {
562                return suffix.parse().unwrap();
563            }
564        }
565
566        NumericSuffix::None
567    }
568
569    /// Is this token the beginning of a variable/function declaration?
570    /// If so, what kind?
571    /// If not, returns None.
572    pub fn declaration_keyword(&self) -> Option<VariableKind> {
573        if !matches!(self.token_type, TokenType::Keyword) {
574            return None;
575        }
576        Some(match self.value.as_str() {
577            "fn" => VariableKind::Fn,
578            "var" | "let" | "const" => VariableKind::Const,
579            _ => return None,
580        })
581    }
582}
583
584impl From<Token> for SourceRange {
585    fn from(token: Token) -> Self {
586        Self::new(token.start, token.end, token.module_id)
587    }
588}
589
590impl From<&Token> for SourceRange {
591    fn from(token: &Token) -> Self {
592        Self::new(token.start, token.end, token.module_id)
593    }
594}
595
596/// Environment variable selecting which lexer implementation [`lex`] uses.
597pub(crate) const KCL_LEXER_ENV_VAR: &str = "KCL_LEXER";
598
599/// Which lexer implementation [`lex`] uses: the old winnow `tokeniser` (`Old`) or
600/// the new `kcl-syntax` logos lexer (`New`). Selected at runtime via the
601/// `KCL_LEXER` environment variable, so a process can pick either lexer without a
602/// rebuild.
603///
604/// Precedence: test override > `KCL_LEXER` > [`LexerMode::DEFAULT`]. This mirrors
605/// the existing `KCL_MEMORY_IMPL` selector in `execution::memory`.
606#[derive(Debug, Clone, Copy, PartialEq, Eq)]
607pub(crate) enum LexerMode {
608    Old,
609    New,
610}
611
612impl LexerMode {
613    /// The mode used when `KCL_LEXER` is unset.
614    const DEFAULT: Self = Self::Old;
615
616    /// Resolve the active lexer mode (see precedence on [`LexerMode`]).
617    pub(crate) fn resolve() -> Self {
618        #[cfg(test)]
619        if let Some(mode) = Self::test_override() {
620            return mode;
621        }
622
623        match env::var(KCL_LEXER_ENV_VAR) {
624            Ok(value) => Self::parse(&value),
625            Err(env::VarError::NotPresent) => Self::DEFAULT,
626            Err(env::VarError::NotUnicode(value)) => {
627                // Invalid-unicode env var: warn and fall back rather than crash.
628                Self::warn_once(|| {
629                    format!(
630                        "{KCL_LEXER_ENV_VAR} must be valid unicode; got `{}`. Defaulting to `old`.",
631                        value.to_string_lossy()
632                    )
633                });
634                Self::Old
635            }
636        }
637    }
638
639    fn parse(value: &str) -> Self {
640        let value = value.trim();
641        if value.eq_ignore_ascii_case("old") {
642            return Self::Old;
643        }
644        if value.eq_ignore_ascii_case("new") {
645            return Self::New;
646        }
647
648        // A mistyped `KCL_LEXER` should not crash the process: warn and fall back
649        // to the old lexer (the conservative choice for a misconfiguration).
650        Self::warn_once(|| {
651            format!("Unsupported {KCL_LEXER_ENV_VAR} value `{value}`; expected `old` or `new`. Defaulting to `old`.")
652        });
653        Self::Old
654    }
655
656    /// Emit a one-time configuration warning through `crate::log` (gated on
657    /// `ZOO_LOG`). `resolve`/`parse` run on every `lex`, so a misconfigured
658    /// `KCL_LEXER` must not warn -- or allocate the message -- on every call. One
659    /// guard suffices: only one kind of misconfiguration can occur per process,
660    /// since the env var holds a single value.
661    fn warn_once(make_message: impl FnOnce() -> String) {
662        static WARNED: std::sync::Once = std::sync::Once::new();
663        WARNED.call_once(|| crate::log::log(make_message()));
664    }
665
666    #[cfg(test)]
667    fn test_override_value(self) -> u8 {
668        match self {
669            Self::Old => 1,
670            Self::New => 2,
671        }
672    }
673
674    #[cfg(test)]
675    fn test_override() -> Option<Self> {
676        match TEST_LEXER_MODE_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
677            1 => Some(Self::Old),
678            2 => Some(Self::New),
679            _ => None,
680        }
681    }
682
683    /// Override the lexer mode for the lifetime of the returned guard.
684    ///
685    /// This uses a process-global atomic, so it is only race-free under test
686    /// runners that isolate tests in separate processes (e.g. `cargo nextest`).
687    /// Under in-process parallel `cargo test`, prefer driving the lexer with an
688    /// explicit mode; reserve this guard for dispatch/integration tests.
689    #[cfg(test)]
690    pub(crate) fn override_for_test(mode: Self) -> LexerModeOverrideGuard {
691        let previous = TEST_LEXER_MODE_OVERRIDE.swap(mode.test_override_value(), std::sync::atomic::Ordering::SeqCst);
692        LexerModeOverrideGuard { previous }
693    }
694}
695
696#[cfg(test)]
697static TEST_LEXER_MODE_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
698
699#[cfg(test)]
700pub(crate) struct LexerModeOverrideGuard {
701    previous: u8,
702}
703
704#[cfg(test)]
705impl Drop for LexerModeOverrideGuard {
706    fn drop(&mut self) {
707        TEST_LEXER_MODE_OVERRIDE.store(self.previous, std::sync::atomic::Ordering::SeqCst);
708    }
709}
710
711// `lex` dispatches on the runtime `LexerMode`. `Old` runs the winnow
712// `tokeniser`; `New` runs the `kcl-syntax` adapter and folds any fatal lexical
713// diagnostics into a single lexical `KclError`, preserving the public `Result`
714// contract. (The LSP consumes the richer `LexResult` directly so it can keep
715// tokens for highlighting while reporting diagnostics.)
716pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
717    match LexerMode::resolve() {
718        LexerMode::Old => lex_legacy(s, module_id),
719        LexerMode::New => {
720            let result = adapter::lex_with_diagnostics(s, module_id);
721            match result.to_lexical_error() {
722                Some(err) => Err(err),
723                None => Ok(result.tokens),
724            }
725        }
726    }
727}
728
729fn lex_legacy(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
730    tokeniser::lex(s, module_id).map_err(|err| {
731        let (input, offset): (Vec<char>, usize) = (err.input().chars().collect(), err.offset());
732        let module_id = err.input().state.module_id;
733
734        if offset >= input.len() {
735            // From the winnow docs:
736            //
737            // This is an offset, not an index, and may point to
738            // the end of input (input.len()) on eof errors.
739
740            return KclError::new_lexical(KclErrorDetails::new(
741                "unexpected EOF while parsing".to_owned(),
742                vec![SourceRange::new(offset, offset, module_id)],
743            ));
744        }
745
746        // TODO: Add the Winnow tokenizer context to the error.
747        // See https://github.com/KittyCAD/modeling-app/issues/784
748        let bad_token = &input[offset];
749        // TODO: Add the Winnow parser context to the error.
750        // See https://github.com/KittyCAD/modeling-app/issues/784
751        KclError::new_lexical(KclErrorDetails::new(
752            format!("found unknown token '{bad_token}'"),
753            vec![SourceRange::new(offset, offset + 1, module_id)],
754        ))
755    })
756}
757
758#[cfg(test)]
759mod lexer_mode_tests {
760    use super::LexerMode;
761    use super::lex;
762    use crate::ModuleId;
763
764    #[test]
765    fn default_mode_is_old() {
766        assert_eq!(LexerMode::DEFAULT, LexerMode::Old);
767    }
768
769    #[test]
770    fn parse_accepts_known_values_case_insensitively() {
771        assert_eq!(LexerMode::parse("old"), LexerMode::Old);
772        assert_eq!(LexerMode::parse("  NEW  "), LexerMode::New);
773    }
774
775    #[test]
776    fn parse_falls_back_to_old_on_unknown_value() {
777        // An unknown value warns and defaults to the old lexer instead of panicking.
778        assert_eq!(LexerMode::parse("rowan"), LexerMode::Old);
779    }
780
781    #[test]
782    fn override_guard_sets_and_restores_mode() {
783        // Reserved for dispatch/integration tests; relies on the process-global
784        // atomic, which is race-free under nextest's process isolation.
785        {
786            let _guard = LexerMode::override_for_test(LexerMode::New);
787            assert_eq!(LexerMode::resolve(), LexerMode::New);
788        }
789        let _guard = LexerMode::override_for_test(LexerMode::Old);
790        assert_eq!(LexerMode::resolve(), LexerMode::Old);
791    }
792
793    /// Exercises the `New` arm of `lex` in default CI: no `KCL_LEXER` env var is
794    /// set; the new lexer is selected via the process-global test override (which
795    /// is race-free under nextest's process-per-test isolation).
796    ///
797    /// The unterminated-string assertion is deliberately a *distinguishing* one:
798    /// the new lexer folds the recovery token into the message "unterminated
799    /// string literal", whereas the old lexer reports `found unknown token '"'`.
800    /// Asserting the new-lexer-only message proves `lex` took the `New` arm --
801    /// not merely that some lexer ran.
802    #[test]
803    fn lex_dispatches_to_new_lexer() {
804        let _guard = LexerMode::override_for_test(LexerMode::New);
805        assert_eq!(LexerMode::resolve(), LexerMode::New);
806
807        let module_id = ModuleId::default();
808
809        // Valid input flows through the New arm and yields a token stream.
810        let tokens = lex("x = 1", module_id).expect("new lexer should tokenize valid input");
811        assert!(!tokens.is_empty(), "expected a non-empty token stream");
812
813        // Unterminated string: the new-lexer-only message (see doc comment).
814        let err = lex("\"abc", module_id).expect_err("unterminated string is a lexical error");
815        assert_eq!(err.error_type(), "lexical");
816        assert_eq!(err.message(), "unterminated string literal");
817    }
818}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
