kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::fmt;
5use std::iter::Enumerate;
6use std::num::NonZeroUsize;
7use std::str::FromStr;
8
9use anyhow::Result;
10use kcl_error::KclErrorDetails;
11use parse_display::Display;
12use serde::Deserialize;
13use serde::Serialize;
14use tower_lsp::lsp_types::SemanticTokenType;
15use winnow::stream::ContainsToken;
16use winnow::stream::Stream;
17use winnow::{self};
18
19use crate::CompilationIssue;
20use crate::ModuleId;
21use crate::SourceRange;
22use crate::errors::KclError;
23use crate::parsing::ast::types::ItemVisibility;
24use crate::parsing::ast::types::VariableKind;
25
26mod tokeniser;
27
28#[cfg(all(test, feature = "new-scanner"))]
29mod compat_tests;
30
31pub(crate) use tokeniser::RESERVED_SKETCH_BLOCK_WORDS;
32pub(crate) use tokeniser::RESERVED_WORDS;
33
34// Note the ordering, it's important that `m` comes after `mm` and `cm`.
35pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
36
37#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
38#[repr(u32)]
39pub enum NumericSuffix {
40    None,
41    Count,
42    Length,
43    Angle,
44    Mm,
45    Cm,
46    M,
47    Inch,
48    Ft,
49    Yd,
50    Deg,
51    Rad,
52    Unknown,
53}
54
55impl NumericSuffix {
56    #[allow(dead_code)]
57    pub fn is_none(self) -> bool {
58        self == Self::None
59    }
60
61    pub fn is_some(self) -> bool {
62        self != Self::None
63    }
64
65    pub fn digestable_id(&self) -> &[u8] {
66        match self {
67            NumericSuffix::None => &[],
68            NumericSuffix::Count => b"_",
69            NumericSuffix::Unknown => b"?",
70            NumericSuffix::Length => b"Length",
71            NumericSuffix::Angle => b"Angle",
72            NumericSuffix::Mm => b"mm",
73            NumericSuffix::Cm => b"cm",
74            NumericSuffix::M => b"m",
75            NumericSuffix::Inch => b"in",
76            NumericSuffix::Ft => b"ft",
77            NumericSuffix::Yd => b"yd",
78            NumericSuffix::Deg => b"deg",
79            NumericSuffix::Rad => b"rad",
80        }
81    }
82}
83
84impl FromStr for NumericSuffix {
85    type Err = CompilationIssue;
86
87    fn from_str(s: &str) -> Result<Self, Self::Err> {
88        match s {
89            "_" | "Count" => Ok(NumericSuffix::Count),
90            "Length" => Ok(NumericSuffix::Length),
91            "Angle" => Ok(NumericSuffix::Angle),
92            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
93            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
94            "m" | "meters" => Ok(NumericSuffix::M),
95            "inch" | "in" => Ok(NumericSuffix::Inch),
96            "ft" | "feet" => Ok(NumericSuffix::Ft),
97            "yd" | "yards" => Ok(NumericSuffix::Yd),
98            "deg" | "degrees" => Ok(NumericSuffix::Deg),
99            "rad" | "radians" => Ok(NumericSuffix::Rad),
100            "?" => Ok(NumericSuffix::Unknown),
101            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
102        }
103    }
104}
105
106impl fmt::Display for NumericSuffix {
107    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
108        match self {
109            NumericSuffix::None => Ok(()),
110            NumericSuffix::Count => write!(f, "_"),
111            NumericSuffix::Unknown => write!(f, "_?"),
112            NumericSuffix::Length => write!(f, "Length"),
113            NumericSuffix::Angle => write!(f, "Angle"),
114            NumericSuffix::Mm => write!(f, "mm"),
115            NumericSuffix::Cm => write!(f, "cm"),
116            NumericSuffix::M => write!(f, "m"),
117            NumericSuffix::Inch => write!(f, "in"),
118            NumericSuffix::Ft => write!(f, "ft"),
119            NumericSuffix::Yd => write!(f, "yd"),
120            NumericSuffix::Deg => write!(f, "deg"),
121            NumericSuffix::Rad => write!(f, "rad"),
122        }
123    }
124}
125
126#[derive(Clone, Debug, PartialEq)]
127pub(crate) struct TokenStream {
128    tokens: Vec<Token>,
129}
130
131impl TokenStream {
132    fn new(tokens: Vec<Token>) -> Self {
133        Self { tokens }
134    }
135
136    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
137        let tokens = std::mem::take(&mut self.tokens);
138        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
139            .into_iter()
140            .partition(|token| token.token_type != TokenType::Unknown);
141        self.tokens = tokens;
142        unknown_tokens
143    }
144
145    pub fn iter(&self) -> impl Iterator<Item = &Token> {
146        self.tokens.iter()
147    }
148
149    pub fn is_empty(&self) -> bool {
150        self.tokens.is_empty()
151    }
152
153    pub fn as_slice(&self) -> TokenSlice<'_> {
154        TokenSlice::from(self)
155    }
156}
157
158impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
159    fn from(stream: &'a TokenStream) -> Self {
160        TokenSlice {
161            start: 0,
162            end: stream.tokens.len(),
163            stream,
164        }
165    }
166}
167
168impl IntoIterator for TokenStream {
169    type Item = Token;
170
171    type IntoIter = std::vec::IntoIter<Token>;
172
173    fn into_iter(self) -> Self::IntoIter {
174        self.tokens.into_iter()
175    }
176}
177
178#[derive(Debug, Clone)]
179pub(crate) struct TokenSlice<'a> {
180    stream: &'a TokenStream,
181    /// Current position of the leading Token in the stream
182    start: usize,
183    /// The number of total Tokens in the stream
184    end: usize,
185}
186
187impl<'a> std::ops::Deref for TokenSlice<'a> {
188    type Target = [Token];
189
190    fn deref(&self) -> &Self::Target {
191        &self.stream.tokens[self.start..self.end]
192    }
193}
194
195impl<'a> TokenSlice<'a> {
196    pub fn token(&self, i: usize) -> &Token {
197        &self.stream.tokens[i + self.start]
198    }
199
200    pub fn iter(&self) -> impl Iterator<Item = &Token> {
201        (**self).iter()
202    }
203
204    pub fn without_ends(&self) -> Self {
205        Self {
206            start: self.start + 1,
207            end: self.end - 1,
208            stream: self.stream,
209        }
210    }
211
212    pub fn as_source_range(&self) -> SourceRange {
213        let stream_len = self.stream.tokens.len();
214        let first_token = if stream_len == self.start {
215            &self.stream.tokens[self.start - 1]
216        } else {
217            self.token(0)
218        };
219        let last_token = if stream_len == self.end {
220            &self.stream.tokens[stream_len - 1]
221        } else {
222            self.token(self.end - self.start)
223        };
224        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
225    }
226}
227
228impl<'a> IntoIterator for TokenSlice<'a> {
229    type Item = &'a Token;
230
231    type IntoIter = std::slice::Iter<'a, Token>;
232
233    fn into_iter(self) -> Self::IntoIter {
234        self.stream.tokens[self.start..self.end].iter()
235    }
236}
237
238impl<'a> Stream for TokenSlice<'a> {
239    type Token = Token;
240    type Slice = Self;
241    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
242    type Checkpoint = Checkpoint;
243
244    fn iter_offsets(&self) -> Self::IterOffsets {
245        #[allow(clippy::unnecessary_to_owned)]
246        self.to_vec().into_iter().enumerate()
247    }
248
249    fn eof_offset(&self) -> usize {
250        self.len()
251    }
252
253    fn next_token(&mut self) -> Option<Self::Token> {
254        let token = self.first()?.clone();
255        self.start += 1;
256        Some(token)
257    }
258
259    /// Split off the next token from the input
260    fn peek_token(&self) -> Option<Self::Token> {
261        Some(self.first()?.clone())
262    }
263
264    fn offset_for<P>(&self, predicate: P) -> Option<usize>
265    where
266        P: Fn(Self::Token) -> bool,
267    {
268        self.iter().position(|b| predicate(b.clone()))
269    }
270
271    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
272        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
273            Err(winnow::error::Needed::Size(needed))
274        } else {
275            Ok(tokens)
276        }
277    }
278
279    fn next_slice(&mut self, offset: usize) -> Self::Slice {
280        assert!(self.start + offset <= self.end);
281
282        let next = TokenSlice {
283            stream: self.stream,
284            start: self.start,
285            end: self.start + offset,
286        };
287        self.start += offset;
288        next
289    }
290
291    /// Split off a slice of tokens from the input
292    fn peek_slice(&self, offset: usize) -> Self::Slice {
293        assert!(self.start + offset <= self.end);
294
295        TokenSlice {
296            stream: self.stream,
297            start: self.start,
298            end: self.start + offset,
299        }
300    }
301
302    fn checkpoint(&self) -> Self::Checkpoint {
303        Checkpoint(self.start, self.end)
304    }
305
306    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
307        self.start = checkpoint.0;
308        self.end = checkpoint.1;
309    }
310
311    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
312        write!(f, "{self:?}")
313    }
314}
315
316impl<'a> winnow::stream::Offset for TokenSlice<'a> {
317    fn offset_from(&self, start: &Self) -> usize {
318        self.start - start.start
319    }
320}
321
322impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
323    fn offset_from(&self, start: &Checkpoint) -> usize {
324        self.start - start.0
325    }
326}
327
328impl winnow::stream::Offset for Checkpoint {
329    fn offset_from(&self, start: &Self) -> usize {
330        self.0 - start.0
331    }
332}
333
334impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
335    type PartialState = ();
336
337    fn complete(&mut self) -> Self::PartialState {}
338
339    fn restore_partial(&mut self, _: Self::PartialState) {}
340
341    fn is_partial_supported() -> bool {
342        false
343    }
344}
345
346impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
347    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
348        self.iter()
349            .enumerate()
350            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
351    }
352}
353
354#[derive(Clone, Debug)]
355pub struct Checkpoint(usize, usize);
356
357/// The types of tokens.
358#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
359#[display(style = "camelCase")]
360pub enum TokenType {
361    /// A number.
362    Number,
363    /// A word.
364    Word,
365    /// An operator.
366    Operator,
367    /// A string.
368    String,
369    /// A keyword.
370    Keyword,
371    /// A type.
372    Type,
373    /// A brace.
374    Brace,
375    /// A hash.
376    Hash,
377    /// A bang.
378    Bang,
379    /// A dollar sign.
380    Dollar,
381    /// Whitespace.
382    Whitespace,
383    /// A comma.
384    Comma,
385    /// A colon.
386    Colon,
387    /// A double colon: `::`
388    DoubleColon,
389    /// A period.
390    Period,
391    /// A double period: `..`.
392    DoublePeriod,
393    /// A double period and a less than: `..<`.
394    DoublePeriodLessThan,
395    /// A line comment.
396    LineComment,
397    /// A block comment.
398    BlockComment,
399    /// A function name.
400    Function,
401    /// Unknown lexemes.
402    Unknown,
403    /// The ? symbol, used for optional values.
404    QuestionMark,
405    /// The @ symbol.
406    At,
407    /// `;`
408    SemiColon,
409}
410
411/// Most KCL tokens correspond to LSP semantic tokens (but not all).
412impl TryFrom<TokenType> for SemanticTokenType {
413    type Error = anyhow::Error;
414    fn try_from(token_type: TokenType) -> Result<Self> {
415        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
416        // in the LSP implementation.
417        Ok(match token_type {
418            TokenType::Number => Self::NUMBER,
419            TokenType::Word => Self::VARIABLE,
420            TokenType::Keyword => Self::KEYWORD,
421            TokenType::Type => Self::TYPE,
422            TokenType::Operator => Self::OPERATOR,
423            TokenType::QuestionMark => Self::OPERATOR,
424            TokenType::String => Self::STRING,
425            TokenType::Bang => Self::OPERATOR,
426            TokenType::LineComment => Self::COMMENT,
427            TokenType::BlockComment => Self::COMMENT,
428            TokenType::Function => Self::FUNCTION,
429            TokenType::Whitespace
430            | TokenType::Brace
431            | TokenType::Comma
432            | TokenType::Colon
433            | TokenType::DoubleColon
434            | TokenType::Period
435            | TokenType::DoublePeriod
436            | TokenType::DoublePeriodLessThan
437            | TokenType::Hash
438            | TokenType::Dollar
439            | TokenType::At
440            | TokenType::SemiColon
441            | TokenType::Unknown => {
442                anyhow::bail!("unsupported token type: {:?}", token_type)
443            }
444        })
445    }
446}
447
448impl TokenType {
449    pub fn is_whitespace(&self) -> bool {
450        matches!(self, Self::Whitespace)
451    }
452
453    pub fn is_comment(&self) -> bool {
454        matches!(self, Self::LineComment | Self::BlockComment)
455    }
456}
457
458#[derive(Debug, PartialEq, Eq, Clone)]
459pub struct Token {
460    pub token_type: TokenType,
461    /// Offset in the source code where this token begins.
462    pub start: usize,
463    /// Offset in the source code where this token ends.
464    pub end: usize,
465    pub(super) module_id: ModuleId,
466    pub(super) value: String,
467}
468
469impl ContainsToken<Token> for (TokenType, &str) {
470    fn contains_token(&self, token: Token) -> bool {
471        self.0 == token.token_type && self.1 == token.value
472    }
473}
474
475impl ContainsToken<Token> for TokenType {
476    fn contains_token(&self, token: Token) -> bool {
477        *self == token.token_type
478    }
479}
480
481impl Token {
482    pub fn from_range(
483        range: std::ops::Range<usize>,
484        module_id: ModuleId,
485        token_type: TokenType,
486        value: String,
487    ) -> Self {
488        Self {
489            start: range.start,
490            end: range.end,
491            module_id,
492            value,
493            token_type,
494        }
495    }
496    pub fn is_code_token(&self) -> bool {
497        !matches!(
498            self.token_type,
499            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
500        )
501    }
502
503    pub fn as_source_range(&self) -> SourceRange {
504        SourceRange::new(self.start, self.end, self.module_id)
505    }
506
507    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
508        vec![self.as_source_range()]
509    }
510
511    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
512        if !matches!(self.token_type, TokenType::Keyword) {
513            return None;
514        }
515        match self.value.as_str() {
516            "export" => Some(ItemVisibility::Export),
517            _ => None,
518        }
519    }
520
521    pub fn numeric_value(&self) -> Option<f64> {
522        if self.token_type != TokenType::Number {
523            return None;
524        }
525        let value = &self.value;
526        let value = value
527            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
528            .map(|(s, _)| s)
529            .unwrap_or(value);
530        value.parse().ok()
531    }
532
533    pub fn uint_value(&self) -> Option<u32> {
534        if self.token_type != TokenType::Number {
535            return None;
536        }
537        let value = &self.value;
538        let value = value
539            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
540            .map(|(s, _)| s)
541            .unwrap_or(value);
542        value.parse().ok()
543    }
544
545    pub fn numeric_suffix(&self) -> NumericSuffix {
546        if self.token_type != TokenType::Number {
547            return NumericSuffix::None;
548        }
549
550        if self.value.ends_with('_') {
551            return NumericSuffix::Count;
552        }
553
554        for suffix in NUM_SUFFIXES {
555            if self.value.ends_with(suffix) {
556                return suffix.parse().unwrap();
557            }
558        }
559
560        NumericSuffix::None
561    }
562
563    /// Is this token the beginning of a variable/function declaration?
564    /// If so, what kind?
565    /// If not, returns None.
566    pub fn declaration_keyword(&self) -> Option<VariableKind> {
567        if !matches!(self.token_type, TokenType::Keyword) {
568            return None;
569        }
570        Some(match self.value.as_str() {
571            "fn" => VariableKind::Fn,
572            "var" | "let" | "const" => VariableKind::Const,
573            _ => return None,
574        })
575    }
576}
577
578impl From<Token> for SourceRange {
579    fn from(token: Token) -> Self {
580        Self::new(token.start, token.end, token.module_id)
581    }
582}
583
584impl From<&Token> for SourceRange {
585    fn from(token: &Token) -> Self {
586        Self::new(token.start, token.end, token.module_id)
587    }
588}
589
590pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
591    tokeniser::lex(s, module_id).map_err(|err| {
592        let (input, offset): (Vec<char>, usize) = (err.input().chars().collect(), err.offset());
593        let module_id = err.input().state.module_id;
594
595        if offset >= input.len() {
596            // From the winnow docs:
597            //
598            // This is an offset, not an index, and may point to
599            // the end of input (input.len()) on eof errors.
600
601            return KclError::new_lexical(KclErrorDetails::new(
602                "unexpected EOF while parsing".to_owned(),
603                vec![SourceRange::new(offset, offset, module_id)],
604            ));
605        }
606
607        // TODO: Add the Winnow tokenizer context to the error.
608        // See https://github.com/KittyCAD/modeling-app/issues/784
609        let bad_token = &input[offset];
610        // TODO: Add the Winnow parser context to the error.
611        // See https://github.com/KittyCAD/modeling-app/issues/784
612        KclError::new_lexical(KclErrorDetails::new(
613            format!("found unknown token '{bad_token}'"),
614            vec![SourceRange::new(offset, offset + 1, module_id)],
615        ))
616    })
617}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
