kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::fmt;
5use std::iter::Enumerate;
6use std::num::NonZeroUsize;
7use std::str::FromStr;
8
9use anyhow::Result;
10use parse_display::Display;
11use serde::Deserialize;
12use serde::Serialize;
13use tower_lsp::lsp_types::SemanticTokenType;
14use winnow::stream::ContainsToken;
15use winnow::stream::Stream;
16use winnow::{self};
17
18use crate::CompilationIssue;
19use crate::ModuleId;
20use crate::SourceRange;
21use crate::errors::KclError;
22use crate::parsing::ast::types::ItemVisibility;
23use crate::parsing::ast::types::VariableKind;
24
25#[doc(hidden)]
26pub mod adapter;
27
28pub use kcl_syntax::keywords::KEYWORDS as RESERVED_WORDS;
29
30// Note the ordering, it's important that `m` comes after `mm` and `cm`.
31pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
32
33#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
34#[repr(u32)]
35pub enum NumericSuffix {
36    None,
37    Count,
38    Length,
39    Angle,
40    Mm,
41    Cm,
42    M,
43    Inch,
44    Ft,
45    Yd,
46    Deg,
47    Rad,
48    Unknown,
49}
50
51impl NumericSuffix {
52    #[allow(dead_code)]
53    pub fn is_none(self) -> bool {
54        self == Self::None
55    }
56
57    pub fn is_some(self) -> bool {
58        self != Self::None
59    }
60
61    pub fn digestable_id(&self) -> &[u8] {
62        match self {
63            NumericSuffix::None => &[],
64            NumericSuffix::Count => b"_",
65            NumericSuffix::Unknown => b"?",
66            NumericSuffix::Length => b"Length",
67            NumericSuffix::Angle => b"Angle",
68            NumericSuffix::Mm => b"mm",
69            NumericSuffix::Cm => b"cm",
70            NumericSuffix::M => b"m",
71            NumericSuffix::Inch => b"in",
72            NumericSuffix::Ft => b"ft",
73            NumericSuffix::Yd => b"yd",
74            NumericSuffix::Deg => b"deg",
75            NumericSuffix::Rad => b"rad",
76        }
77    }
78}
79
80impl FromStr for NumericSuffix {
81    type Err = CompilationIssue;
82
83    fn from_str(s: &str) -> Result<Self, Self::Err> {
84        match s {
85            "_" | "Count" => Ok(NumericSuffix::Count),
86            "Length" => Ok(NumericSuffix::Length),
87            "Angle" => Ok(NumericSuffix::Angle),
88            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
89            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
90            "m" | "meters" => Ok(NumericSuffix::M),
91            "inch" | "in" => Ok(NumericSuffix::Inch),
92            "ft" | "feet" => Ok(NumericSuffix::Ft),
93            "yd" | "yards" => Ok(NumericSuffix::Yd),
94            "deg" | "degrees" => Ok(NumericSuffix::Deg),
95            "rad" | "radians" => Ok(NumericSuffix::Rad),
96            "?" => Ok(NumericSuffix::Unknown),
97            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
98        }
99    }
100}
101
102impl fmt::Display for NumericSuffix {
103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104        match self {
105            NumericSuffix::None => Ok(()),
106            NumericSuffix::Count => write!(f, "_"),
107            NumericSuffix::Unknown => write!(f, "_?"),
108            NumericSuffix::Length => write!(f, "Length"),
109            NumericSuffix::Angle => write!(f, "Angle"),
110            NumericSuffix::Mm => write!(f, "mm"),
111            NumericSuffix::Cm => write!(f, "cm"),
112            NumericSuffix::M => write!(f, "m"),
113            NumericSuffix::Inch => write!(f, "in"),
114            NumericSuffix::Ft => write!(f, "ft"),
115            NumericSuffix::Yd => write!(f, "yd"),
116            NumericSuffix::Deg => write!(f, "deg"),
117            NumericSuffix::Rad => write!(f, "rad"),
118        }
119    }
120}
121
122#[derive(Clone, Debug, PartialEq)]
123pub struct TokenStream {
124    tokens: Vec<Token>,
125}
126
127impl TokenStream {
128    fn new(tokens: Vec<Token>) -> Self {
129        Self { tokens }
130    }
131
132    /// Allow the parser to read `use` as an identifier until the module's KCL
133    /// version is known. Return the original keyword ranges for validation.
134    pub(super) fn allow_use_identifiers(&mut self) -> Vec<SourceRange> {
135        self.tokens
136            .iter_mut()
137            .filter_map(|token| {
138                if token.token_type == TokenType::Keyword && token.value == "use" {
139                    token.token_type = TokenType::Word;
140                    Some(token.as_source_range())
141                } else {
142                    None
143                }
144            })
145            .collect()
146    }
147
148    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
149        let tokens = std::mem::take(&mut self.tokens);
150        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
151            .into_iter()
152            .partition(|token| token.token_type != TokenType::Unknown);
153        self.tokens = tokens;
154        unknown_tokens
155    }
156
157    pub fn iter(&self) -> impl Iterator<Item = &Token> {
158        self.tokens.iter()
159    }
160
161    pub fn is_empty(&self) -> bool {
162        self.tokens.is_empty()
163    }
164
165    pub fn as_slice(&self) -> TokenSlice<'_> {
166        TokenSlice::from(self)
167    }
168}
169
170impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
171    fn from(stream: &'a TokenStream) -> Self {
172        TokenSlice {
173            start: 0,
174            end: stream.tokens.len(),
175            stream,
176        }
177    }
178}
179
180impl IntoIterator for TokenStream {
181    type Item = Token;
182
183    type IntoIter = std::vec::IntoIter<Token>;
184
185    fn into_iter(self) -> Self::IntoIter {
186        self.tokens.into_iter()
187    }
188}
189
190#[derive(Debug, Clone)]
191pub struct TokenSlice<'a> {
192    stream: &'a TokenStream,
193    /// Current position of the leading Token in the stream
194    start: usize,
195    /// The number of total Tokens in the stream
196    end: usize,
197}
198
199impl<'a> std::ops::Deref for TokenSlice<'a> {
200    type Target = [Token];
201
202    fn deref(&self) -> &Self::Target {
203        &self.stream.tokens[self.start..self.end]
204    }
205}
206
207impl<'a> TokenSlice<'a> {
208    pub fn token(&self, i: usize) -> &Token {
209        &self.stream.tokens[i + self.start]
210    }
211
212    pub fn iter(&self) -> impl Iterator<Item = &Token> {
213        (**self).iter()
214    }
215
216    pub fn without_ends(&self) -> Self {
217        Self {
218            start: self.start + 1,
219            end: self.end - 1,
220            stream: self.stream,
221        }
222    }
223
224    pub fn as_source_range(&self) -> SourceRange {
225        let stream_len = self.stream.tokens.len();
226        let first_token = if stream_len == self.start {
227            &self.stream.tokens[self.start - 1]
228        } else {
229            self.token(0)
230        };
231        let last_token = if stream_len == self.end {
232            &self.stream.tokens[stream_len - 1]
233        } else {
234            self.token(self.end - self.start)
235        };
236        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
237    }
238}
239
240impl<'a> IntoIterator for TokenSlice<'a> {
241    type Item = &'a Token;
242
243    type IntoIter = std::slice::Iter<'a, Token>;
244
245    fn into_iter(self) -> Self::IntoIter {
246        self.stream.tokens[self.start..self.end].iter()
247    }
248}
249
250impl<'a> Stream for TokenSlice<'a> {
251    type Token = Token;
252    type Slice = Self;
253    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
254    type Checkpoint = Checkpoint;
255
256    fn iter_offsets(&self) -> Self::IterOffsets {
257        #[allow(clippy::unnecessary_to_owned)]
258        self.to_vec().into_iter().enumerate()
259    }
260
261    fn eof_offset(&self) -> usize {
262        self.len()
263    }
264
265    fn next_token(&mut self) -> Option<Self::Token> {
266        let token = self.first()?.clone();
267        self.start += 1;
268        Some(token)
269    }
270
271    /// Split off the next token from the input
272    fn peek_token(&self) -> Option<Self::Token> {
273        Some(self.first()?.clone())
274    }
275
276    fn offset_for<P>(&self, predicate: P) -> Option<usize>
277    where
278        P: Fn(Self::Token) -> bool,
279    {
280        self.iter().position(|b| predicate(b.clone()))
281    }
282
283    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
284        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
285            Err(winnow::error::Needed::Size(needed))
286        } else {
287            Ok(tokens)
288        }
289    }
290
291    fn next_slice(&mut self, offset: usize) -> Self::Slice {
292        assert!(self.start + offset <= self.end);
293
294        let next = TokenSlice {
295            stream: self.stream,
296            start: self.start,
297            end: self.start + offset,
298        };
299        self.start += offset;
300        next
301    }
302
303    /// Split off a slice of tokens from the input
304    fn peek_slice(&self, offset: usize) -> Self::Slice {
305        assert!(self.start + offset <= self.end);
306
307        TokenSlice {
308            stream: self.stream,
309            start: self.start,
310            end: self.start + offset,
311        }
312    }
313
314    fn checkpoint(&self) -> Self::Checkpoint {
315        Checkpoint(self.start, self.end)
316    }
317
318    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
319        self.start = checkpoint.0;
320        self.end = checkpoint.1;
321    }
322
323    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
324        write!(f, "{self:?}")
325    }
326}
327
328impl<'a> winnow::stream::Offset for TokenSlice<'a> {
329    fn offset_from(&self, start: &Self) -> usize {
330        self.start - start.start
331    }
332}
333
334impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
335    fn offset_from(&self, start: &Checkpoint) -> usize {
336        self.start - start.0
337    }
338}
339
340impl winnow::stream::Offset for Checkpoint {
341    fn offset_from(&self, start: &Self) -> usize {
342        self.0 - start.0
343    }
344}
345
346impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
347    type PartialState = ();
348
349    fn complete(&mut self) -> Self::PartialState {}
350
351    fn restore_partial(&mut self, _: Self::PartialState) {}
352
353    fn is_partial_supported() -> bool {
354        false
355    }
356}
357
358impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
359    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
360        self.iter()
361            .enumerate()
362            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
363    }
364}
365
366#[derive(Clone, Debug)]
367pub struct Checkpoint(usize, usize);
368
369/// The types of tokens.
370#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
371#[display(style = "camelCase")]
372pub enum TokenType {
373    /// A number.
374    Number,
375    /// A word.
376    Word,
377    /// An operator.
378    Operator,
379    /// A string.
380    String,
381    /// A keyword.
382    Keyword,
383    /// A type.
384    Type,
385    /// A brace.
386    Brace,
387    /// A hash.
388    Hash,
389    /// A bang.
390    Bang,
391    /// A dollar sign.
392    Dollar,
393    /// Whitespace.
394    Whitespace,
395    /// A comma.
396    Comma,
397    /// A colon.
398    Colon,
399    /// A double colon: `::`
400    DoubleColon,
401    /// A period.
402    Period,
403    /// A double period: `..`.
404    DoublePeriod,
405    /// A double period and a less than: `..<`.
406    DoublePeriodLessThan,
407    /// A line comment.
408    LineComment,
409    /// A block comment.
410    BlockComment,
411    /// A function name.
412    Function,
413    /// Unknown lexemes.
414    Unknown,
415    /// The ? symbol, used for optional values.
416    QuestionMark,
417    /// The @ symbol.
418    At,
419    /// `;`
420    SemiColon,
421}
422
423/// Most KCL tokens correspond to LSP semantic tokens (but not all).
424impl TryFrom<TokenType> for SemanticTokenType {
425    type Error = anyhow::Error;
426    fn try_from(token_type: TokenType) -> Result<Self> {
427        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
428        // in the LSP implementation.
429        Ok(match token_type {
430            TokenType::Number => Self::NUMBER,
431            TokenType::Word => Self::VARIABLE,
432            TokenType::Keyword => Self::KEYWORD,
433            TokenType::Type => Self::TYPE,
434            TokenType::Operator => Self::OPERATOR,
435            TokenType::QuestionMark => Self::OPERATOR,
436            TokenType::String => Self::STRING,
437            TokenType::Bang => Self::OPERATOR,
438            TokenType::LineComment => Self::COMMENT,
439            TokenType::BlockComment => Self::COMMENT,
440            TokenType::Function => Self::FUNCTION,
441            TokenType::Whitespace
442            | TokenType::Brace
443            | TokenType::Comma
444            | TokenType::Colon
445            | TokenType::DoubleColon
446            | TokenType::Period
447            | TokenType::DoublePeriod
448            | TokenType::DoublePeriodLessThan
449            | TokenType::Hash
450            | TokenType::Dollar
451            | TokenType::At
452            | TokenType::SemiColon
453            | TokenType::Unknown => {
454                anyhow::bail!("unsupported token type: {:?}", token_type)
455            }
456        })
457    }
458}
459
460impl TokenType {
461    pub fn is_whitespace(&self) -> bool {
462        matches!(self, Self::Whitespace)
463    }
464
465    pub fn is_comment(&self) -> bool {
466        matches!(self, Self::LineComment | Self::BlockComment)
467    }
468}
469
470#[derive(Debug, PartialEq, Eq, Clone)]
471pub struct Token {
472    pub token_type: TokenType,
473    /// Offset in the source code where this token begins.
474    pub start: usize,
475    /// Offset in the source code where this token ends.
476    pub end: usize,
477    pub(super) module_id: ModuleId,
478    pub(super) value: String,
479}
480
481impl ContainsToken<Token> for (TokenType, &str) {
482    fn contains_token(&self, token: Token) -> bool {
483        self.0 == token.token_type && self.1 == token.value
484    }
485}
486
487impl ContainsToken<Token> for TokenType {
488    fn contains_token(&self, token: Token) -> bool {
489        *self == token.token_type
490    }
491}
492
493impl Token {
494    pub fn from_range(
495        range: std::ops::Range<usize>,
496        module_id: ModuleId,
497        token_type: TokenType,
498        value: String,
499    ) -> Self {
500        Self {
501            start: range.start,
502            end: range.end,
503            module_id,
504            value,
505            token_type,
506        }
507    }
508    pub fn is_code_token(&self) -> bool {
509        !matches!(
510            self.token_type,
511            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
512        )
513    }
514
515    pub fn as_source_range(&self) -> SourceRange {
516        SourceRange::new(self.start, self.end, self.module_id)
517    }
518
519    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
520        vec![self.as_source_range()]
521    }
522
523    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
524        if !matches!(self.token_type, TokenType::Keyword) {
525            return None;
526        }
527        match self.value.as_str() {
528            "export" => Some(ItemVisibility::Export),
529            _ => None,
530        }
531    }
532
533    pub fn numeric_value(&self) -> Option<f64> {
534        if self.token_type != TokenType::Number {
535            return None;
536        }
537        let value = &self.value;
538        let value = value
539            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
540            .map(|(s, _)| s)
541            .unwrap_or(value);
542        value.parse().ok()
543    }
544
545    pub fn uint_value(&self) -> Option<u32> {
546        if self.token_type != TokenType::Number {
547            return None;
548        }
549        let value = &self.value;
550        let value = value
551            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
552            .map(|(s, _)| s)
553            .unwrap_or(value);
554        value.parse().ok()
555    }
556
557    pub fn numeric_suffix(&self) -> NumericSuffix {
558        if self.token_type != TokenType::Number {
559            return NumericSuffix::None;
560        }
561
562        if self.value.ends_with('_') {
563            return NumericSuffix::Count;
564        }
565
566        for suffix in NUM_SUFFIXES {
567            if self.value.ends_with(suffix) {
568                return suffix.parse().unwrap();
569            }
570        }
571
572        NumericSuffix::None
573    }
574
575    /// Is this token the beginning of a variable/function declaration?
576    /// If so, what kind?
577    /// If not, returns None.
578    pub fn declaration_keyword(&self) -> Option<VariableKind> {
579        if !matches!(self.token_type, TokenType::Keyword) {
580            return None;
581        }
582        Some(match self.value.as_str() {
583            "fn" => VariableKind::Fn,
584            "var" | "let" | "const" => VariableKind::Const,
585            _ => return None,
586        })
587    }
588}
589
590impl From<Token> for SourceRange {
591    fn from(token: Token) -> Self {
592        Self::new(token.start, token.end, token.module_id)
593    }
594}
595
596impl From<&Token> for SourceRange {
597    fn from(token: &Token) -> Self {
598        Self::new(token.start, token.end, token.module_id)
599    }
600}
601
602/// Tokenizes source using `kcl-syntax`.
603pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
604    let result = adapter::lex_with_diagnostics(s, module_id);
605    match result.to_lexical_error() {
606        Some(err) => Err(err),
607        None => Ok(result.tokens),
608    }
609}
610
611#[cfg(test)]
612mod tests {
613    use super::Token;
614    use super::TokenSlice;
615    use super::TokenType;
616    use super::lex;
617    use crate::ModuleId;
618
619    #[test]
620    fn lex_uses_syntax_lexer() {
621        let module_id = ModuleId::default();
622        let tokens = lex("x = 1", module_id).expect("valid input should produce tokens");
623        assert!(!tokens.is_empty());
624
625        let err = lex("\"abc", module_id).expect_err("unterminated string is a lexical error");
626        assert_eq!(err.error_type(), "lexical");
627        assert_eq!(err.message(), "unterminated string literal");
628    }
629
630    #[test]
631    fn test_program2() {
632        let program = r#"const part001 = startSketchOn(XY)
633    |> startProfileAt([0.0000000000, 5.0000000000], %)
634    |> line([0.4900857016, -0.0240763666], %)
635
636const part002 = "part002"
637const things = [part001, 0.0]
638let blah = 1
639const foo = false
640let baz = {a: 1, part001: "thing"}
641
642fn ghi = (part001) => {
643  return part001
644}
645
646show(part001)"#;
647        let module_id = ModuleId::from_usize(1);
648        let actual = lex(program, module_id).unwrap();
649        insta::assert_debug_snapshot!(actual.tokens);
650    }
651
652    #[track_caller]
653    fn assert_tokens(expected: &[(TokenType, usize, usize)], actual: TokenSlice) {
654        let mut e = 0;
655        let mut issues = vec![];
656        for a in actual {
657            if expected[e].0 != a.token_type {
658                if a.token_type == TokenType::Whitespace {
659                    continue;
660                }
661                issues.push(format!(
662                    "Type mismatch: expected `{}`, found `{}` (`{a:?}`), at index {e}",
663                    expected[e].0, a.token_type
664                ));
665            }
666
667            if expected[e].1 != a.start || expected[e].2 != a.end {
668                issues.push(format!(
669                    "Source range mismatch: expected {}-{}, found {}-{} (`{a:?}`), at index {e}",
670                    expected[e].1, expected[e].2, a.start, a.end
671                ));
672            }
673
674            e += 1;
675        }
676        if e < expected.len() {
677            issues.push(format!("Expected `{}` tokens, found `{e}`", expected.len()));
678        }
679        assert!(issues.is_empty(), "{}", issues.join("\n"));
680    }
681
682    #[test]
683    fn test_program0() {
684        let program = "const a=5";
685        let module_id = ModuleId::from_usize(1);
686        let actual = lex(program, module_id).unwrap();
687
688        use TokenType::*;
689        assert_tokens(
690            &[(Keyword, 0, 5), (Word, 6, 7), (Operator, 7, 8), (Number, 8, 9)],
691            actual.as_slice(),
692        );
693    }
694
695    #[test]
696    fn test_program1() {
697        let program = "54 + 22500 + 6";
698        let module_id = ModuleId::from_usize(1);
699        let actual = lex(program, module_id).unwrap();
700
701        use TokenType::*;
702        assert_tokens(
703            &[
704                (Number, 0, 2),
705                (Operator, 3, 4),
706                (Number, 5, 10),
707                (Operator, 11, 12),
708                (Number, 13, 14),
709            ],
710            actual.as_slice(),
711        );
712    }
713
714    #[test]
715    fn test_program3() {
716        let program = r#"
717// this is a comment
718const yo = { a: { b: { c: '123' } } }
719
720const key = 'c'
721const things = "things"
722
723// this is also a comment"#;
724        let module_id = ModuleId::from_usize(1);
725        let actual = lex(program, module_id).unwrap();
726
727        use TokenType::*;
728        assert_tokens(
729            &[
730                (Whitespace, 0, 1),
731                (LineComment, 1, 21),
732                (Whitespace, 21, 22),
733                (Keyword, 22, 27),
734                (Whitespace, 27, 28),
735                (Word, 28, 30),
736                (Whitespace, 30, 31),
737                (Operator, 31, 32),
738                (Whitespace, 32, 33),
739                (Brace, 33, 34),
740                (Whitespace, 34, 35),
741                (Word, 35, 36),
742                (Colon, 36, 37),
743                (Whitespace, 37, 38),
744                (Brace, 38, 39),
745                (Whitespace, 39, 40),
746                (Word, 40, 41),
747                (Colon, 41, 42),
748                (Whitespace, 42, 43),
749                (Brace, 43, 44),
750                (Whitespace, 44, 45),
751                (Word, 45, 46),
752                (Colon, 46, 47),
753                (Whitespace, 47, 48),
754                (String, 48, 53),
755                (Whitespace, 53, 54),
756                (Brace, 54, 55),
757                (Whitespace, 55, 56),
758                (Brace, 56, 57),
759                (Whitespace, 57, 58),
760                (Brace, 58, 59),
761                (Whitespace, 59, 61),
762                (Keyword, 61, 66),
763                (Whitespace, 66, 67),
764                (Word, 67, 70),
765                (Whitespace, 70, 71),
766                (Operator, 71, 72),
767                (Whitespace, 72, 73),
768                (String, 73, 76),
769                (Whitespace, 76, 77),
770                (Keyword, 77, 82),
771                (Whitespace, 82, 83),
772                (Word, 83, 89),
773                (Whitespace, 89, 90),
774                (Operator, 90, 91),
775                (Whitespace, 91, 92),
776                (String, 92, 100),
777                (Whitespace, 100, 102),
778                (LineComment, 102, 127),
779            ],
780            actual.as_slice(),
781        );
782    }
783
784    #[test]
785    fn test_program4() {
786        let program = "const myArray = [0..10]";
787        let module_id = ModuleId::from_usize(1);
788        let actual = lex(program, module_id).unwrap();
789
790        use TokenType::*;
791        assert_tokens(
792            &[
793                (Keyword, 0, 5),
794                (Word, 6, 13),
795                (Operator, 14, 15),
796                (Brace, 16, 17),
797                (Number, 17, 18),
798                (DoublePeriod, 18, 20),
799                (Number, 20, 22),
800                (Brace, 22, 23),
801            ],
802            actual.as_slice(),
803        );
804    }
805
806    #[test]
807    fn test_lexer_negative_word() {
808        let module_id = ModuleId::from_usize(1);
809        let actual = lex("-legX", module_id).unwrap();
810
811        use TokenType::*;
812        assert_tokens(&[(Operator, 0, 1), (Word, 1, 5)], actual.as_slice());
813    }
814
815    #[test]
816    fn not_eq() {
817        let module_id = ModuleId::from_usize(1);
818        let actual = lex("!=", module_id).unwrap();
819        let expected = vec![Token {
820            token_type: TokenType::Operator,
821            value: "!=".to_owned(),
822            start: 0,
823            end: 2,
824            module_id,
825        }];
826        assert_eq!(actual.tokens, expected);
827    }
828
829    #[test]
830    fn import_keyword() {
831        let module_id = ModuleId::from_usize(1);
832        let actual = lex("import foo", module_id).unwrap();
833        let expected = Token {
834            token_type: TokenType::Keyword,
835            value: "import".to_owned(),
836            start: 0,
837            end: 6,
838            module_id,
839        };
840        assert_eq!(actual.tokens[0], expected);
841    }
842
843    #[test]
844    fn import_function() {
845        let module_id = ModuleId::from_usize(1);
846        let actual = lex("import(3)", module_id).unwrap();
847        let expected = Token {
848            token_type: TokenType::Word,
849            value: "import".to_owned(),
850            start: 0,
851            end: 6,
852            module_id,
853        };
854        assert_eq!(actual.tokens[0], expected);
855    }
856
857    #[test]
858    fn use_keyword_and_function_name() {
859        let module_id = ModuleId::default();
860        for (source, expected_type) in [
861            ("use", TokenType::Keyword),
862            ("use = 1", TokenType::Keyword),
863            ("use(3)", TokenType::Word),
864            ("use (3)", TokenType::Keyword),
865            ("useful", TokenType::Word),
866        ] {
867            let tokens = lex(source, module_id).unwrap();
868            assert_eq!(tokens.tokens[0].token_type, expected_type, "{source}");
869        }
870    }
871
872    #[test]
873    fn test_is_code_token() {
874        let module_id = ModuleId::default();
875        let actual = lex("foo (4/* comment */ +,2,\"sdfsdf\") // comment", module_id).unwrap();
876        let non_code = [1, 4, 5, 12, 13];
877        for i in 0..14 {
878            if non_code.contains(&i) {
879                assert!(
880                    !actual.tokens[i].is_code_token(),
881                    "failed test {i}: {:?}",
882                    actual.tokens[i],
883                );
884            } else {
885                assert!(
886                    actual.tokens[i].is_code_token(),
887                    "failed test {i}: {:?}",
888                    actual.tokens[i],
889                );
890            }
891        }
892    }
893    #[test]
894    fn test_boolean_literal() {
895        let module_id = ModuleId::default();
896        let actual = lex("true", module_id).unwrap();
897        let expected = Token {
898            token_type: TokenType::Keyword,
899            value: "true".to_owned(),
900            start: 0,
901            end: 4,
902            module_id,
903        };
904        assert_eq!(actual.tokens[0], expected);
905    }
906
907    #[test]
908    fn test_word_starting_with_keyword() {
909        let module_id = ModuleId::default();
910        let actual = lex("truee", module_id).unwrap();
911        let expected = Token {
912            token_type: TokenType::Word,
913            value: "truee".to_owned(),
914            start: 0,
915            end: 5,
916            module_id,
917        };
918        assert_eq!(actual.tokens[0], expected);
919    }
920
921    #[test]
922    fn non_english_identifiers() {
923        let module_id = ModuleId::default();
924        let actual = lex("亞當", module_id).unwrap();
925        let expected = Token {
926            token_type: TokenType::Word,
927            value: "亞當".to_owned(),
928            start: 0,
929            end: 6,
930            module_id,
931        };
932        assert_eq!(actual.tokens[0], expected);
933    }
934}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
