kcl_lib/parsing/token/
mod.rs

1// Clippy does not agree with rustc here for some reason.
2#![allow(clippy::needless_lifetimes)]
3
4use std::fmt;
5use std::iter::Enumerate;
6use std::num::NonZeroUsize;
7use std::str::FromStr;
8
9use anyhow::Result;
10use parse_display::Display;
11use serde::Deserialize;
12use serde::Serialize;
13use tokeniser::Input;
14use tower_lsp::lsp_types::SemanticTokenType;
15use winnow::error::ParseError;
16use winnow::stream::ContainsToken;
17use winnow::stream::Stream;
18use winnow::{self};
19
20use crate::CompilationIssue;
21use crate::ModuleId;
22use crate::SourceRange;
23use crate::errors::KclError;
24use crate::parsing::ast::types::ItemVisibility;
25use crate::parsing::ast::types::VariableKind;
26
27mod tokeniser;
28
29#[cfg(all(test, feature = "new-scanner"))]
30mod compat_tests;
31
32pub(crate) use tokeniser::RESERVED_SKETCH_BLOCK_WORDS;
33pub(crate) use tokeniser::RESERVED_WORDS;
34
35// Note the ordering, it's important that `m` comes after `mm` and `cm`.
36pub const NUM_SUFFIXES: [&str; 10] = ["mm", "cm", "m", "inch", "in", "ft", "yd", "deg", "rad", "?"];
37
38#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, ts_rs::TS)]
39#[repr(u32)]
40pub enum NumericSuffix {
41    None,
42    Count,
43    Length,
44    Angle,
45    Mm,
46    Cm,
47    M,
48    Inch,
49    Ft,
50    Yd,
51    Deg,
52    Rad,
53    Unknown,
54}
55
56impl NumericSuffix {
57    #[allow(dead_code)]
58    pub fn is_none(self) -> bool {
59        self == Self::None
60    }
61
62    pub fn is_some(self) -> bool {
63        self != Self::None
64    }
65
66    pub fn digestable_id(&self) -> &[u8] {
67        match self {
68            NumericSuffix::None => &[],
69            NumericSuffix::Count => b"_",
70            NumericSuffix::Unknown => b"?",
71            NumericSuffix::Length => b"Length",
72            NumericSuffix::Angle => b"Angle",
73            NumericSuffix::Mm => b"mm",
74            NumericSuffix::Cm => b"cm",
75            NumericSuffix::M => b"m",
76            NumericSuffix::Inch => b"in",
77            NumericSuffix::Ft => b"ft",
78            NumericSuffix::Yd => b"yd",
79            NumericSuffix::Deg => b"deg",
80            NumericSuffix::Rad => b"rad",
81        }
82    }
83}
84
85impl FromStr for NumericSuffix {
86    type Err = CompilationIssue;
87
88    fn from_str(s: &str) -> Result<Self, Self::Err> {
89        match s {
90            "_" | "Count" => Ok(NumericSuffix::Count),
91            "Length" => Ok(NumericSuffix::Length),
92            "Angle" => Ok(NumericSuffix::Angle),
93            "mm" | "millimeters" => Ok(NumericSuffix::Mm),
94            "cm" | "centimeters" => Ok(NumericSuffix::Cm),
95            "m" | "meters" => Ok(NumericSuffix::M),
96            "inch" | "in" => Ok(NumericSuffix::Inch),
97            "ft" | "feet" => Ok(NumericSuffix::Ft),
98            "yd" | "yards" => Ok(NumericSuffix::Yd),
99            "deg" | "degrees" => Ok(NumericSuffix::Deg),
100            "rad" | "radians" => Ok(NumericSuffix::Rad),
101            "?" => Ok(NumericSuffix::Unknown),
102            _ => Err(CompilationIssue::err(SourceRange::default(), "invalid unit of measure")),
103        }
104    }
105}
106
107impl fmt::Display for NumericSuffix {
108    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
109        match self {
110            NumericSuffix::None => Ok(()),
111            NumericSuffix::Count => write!(f, "_"),
112            NumericSuffix::Unknown => write!(f, "_?"),
113            NumericSuffix::Length => write!(f, "Length"),
114            NumericSuffix::Angle => write!(f, "Angle"),
115            NumericSuffix::Mm => write!(f, "mm"),
116            NumericSuffix::Cm => write!(f, "cm"),
117            NumericSuffix::M => write!(f, "m"),
118            NumericSuffix::Inch => write!(f, "in"),
119            NumericSuffix::Ft => write!(f, "ft"),
120            NumericSuffix::Yd => write!(f, "yd"),
121            NumericSuffix::Deg => write!(f, "deg"),
122            NumericSuffix::Rad => write!(f, "rad"),
123        }
124    }
125}
126
127#[derive(Clone, Debug, PartialEq)]
128pub(crate) struct TokenStream {
129    tokens: Vec<Token>,
130}
131
132impl TokenStream {
133    fn new(tokens: Vec<Token>) -> Self {
134        Self { tokens }
135    }
136
137    pub(super) fn remove_unknown(&mut self) -> Vec<Token> {
138        let tokens = std::mem::take(&mut self.tokens);
139        let (tokens, unknown_tokens): (Vec<Token>, Vec<Token>) = tokens
140            .into_iter()
141            .partition(|token| token.token_type != TokenType::Unknown);
142        self.tokens = tokens;
143        unknown_tokens
144    }
145
146    pub fn iter(&self) -> impl Iterator<Item = &Token> {
147        self.tokens.iter()
148    }
149
150    pub fn is_empty(&self) -> bool {
151        self.tokens.is_empty()
152    }
153
154    pub fn as_slice(&self) -> TokenSlice<'_> {
155        TokenSlice::from(self)
156    }
157}
158
159impl<'a> From<&'a TokenStream> for TokenSlice<'a> {
160    fn from(stream: &'a TokenStream) -> Self {
161        TokenSlice {
162            start: 0,
163            end: stream.tokens.len(),
164            stream,
165        }
166    }
167}
168
169impl IntoIterator for TokenStream {
170    type Item = Token;
171
172    type IntoIter = std::vec::IntoIter<Token>;
173
174    fn into_iter(self) -> Self::IntoIter {
175        self.tokens.into_iter()
176    }
177}
178
179#[derive(Debug, Clone)]
180pub(crate) struct TokenSlice<'a> {
181    stream: &'a TokenStream,
182    /// Current position of the leading Token in the stream
183    start: usize,
184    /// The number of total Tokens in the stream
185    end: usize,
186}
187
188impl<'a> std::ops::Deref for TokenSlice<'a> {
189    type Target = [Token];
190
191    fn deref(&self) -> &Self::Target {
192        &self.stream.tokens[self.start..self.end]
193    }
194}
195
196impl<'a> TokenSlice<'a> {
197    pub fn token(&self, i: usize) -> &Token {
198        &self.stream.tokens[i + self.start]
199    }
200
201    pub fn iter(&self) -> impl Iterator<Item = &Token> {
202        (**self).iter()
203    }
204
205    pub fn without_ends(&self) -> Self {
206        Self {
207            start: self.start + 1,
208            end: self.end - 1,
209            stream: self.stream,
210        }
211    }
212
213    pub fn as_source_range(&self) -> SourceRange {
214        let stream_len = self.stream.tokens.len();
215        let first_token = if stream_len == self.start {
216            &self.stream.tokens[self.start - 1]
217        } else {
218            self.token(0)
219        };
220        let last_token = if stream_len == self.end {
221            &self.stream.tokens[stream_len - 1]
222        } else {
223            self.token(self.end - self.start)
224        };
225        SourceRange::new(first_token.start, last_token.end, last_token.module_id)
226    }
227}
228
229impl<'a> IntoIterator for TokenSlice<'a> {
230    type Item = &'a Token;
231
232    type IntoIter = std::slice::Iter<'a, Token>;
233
234    fn into_iter(self) -> Self::IntoIter {
235        self.stream.tokens[self.start..self.end].iter()
236    }
237}
238
239impl<'a> Stream for TokenSlice<'a> {
240    type Token = Token;
241    type Slice = Self;
242    type IterOffsets = Enumerate<std::vec::IntoIter<Token>>;
243    type Checkpoint = Checkpoint;
244
245    fn iter_offsets(&self) -> Self::IterOffsets {
246        #[allow(clippy::unnecessary_to_owned)]
247        self.to_vec().into_iter().enumerate()
248    }
249
250    fn eof_offset(&self) -> usize {
251        self.len()
252    }
253
254    fn next_token(&mut self) -> Option<Self::Token> {
255        let token = self.first()?.clone();
256        self.start += 1;
257        Some(token)
258    }
259
260    /// Split off the next token from the input
261    fn peek_token(&self) -> Option<Self::Token> {
262        Some(self.first()?.clone())
263    }
264
265    fn offset_for<P>(&self, predicate: P) -> Option<usize>
266    where
267        P: Fn(Self::Token) -> bool,
268    {
269        self.iter().position(|b| predicate(b.clone()))
270    }
271
272    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
273        if let Some(needed) = tokens.checked_sub(self.len()).and_then(NonZeroUsize::new) {
274            Err(winnow::error::Needed::Size(needed))
275        } else {
276            Ok(tokens)
277        }
278    }
279
280    fn next_slice(&mut self, offset: usize) -> Self::Slice {
281        assert!(self.start + offset <= self.end);
282
283        let next = TokenSlice {
284            stream: self.stream,
285            start: self.start,
286            end: self.start + offset,
287        };
288        self.start += offset;
289        next
290    }
291
292    /// Split off a slice of tokens from the input
293    fn peek_slice(&self, offset: usize) -> Self::Slice {
294        assert!(self.start + offset <= self.end);
295
296        TokenSlice {
297            stream: self.stream,
298            start: self.start,
299            end: self.start + offset,
300        }
301    }
302
303    fn checkpoint(&self) -> Self::Checkpoint {
304        Checkpoint(self.start, self.end)
305    }
306
307    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
308        self.start = checkpoint.0;
309        self.end = checkpoint.1;
310    }
311
312    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
313        write!(f, "{self:?}")
314    }
315}
316
317impl<'a> winnow::stream::Offset for TokenSlice<'a> {
318    fn offset_from(&self, start: &Self) -> usize {
319        self.start - start.start
320    }
321}
322
323impl<'a> winnow::stream::Offset<Checkpoint> for TokenSlice<'a> {
324    fn offset_from(&self, start: &Checkpoint) -> usize {
325        self.start - start.0
326    }
327}
328
329impl winnow::stream::Offset for Checkpoint {
330    fn offset_from(&self, start: &Self) -> usize {
331        self.0 - start.0
332    }
333}
334
335impl<'a> winnow::stream::StreamIsPartial for TokenSlice<'a> {
336    type PartialState = ();
337
338    fn complete(&mut self) -> Self::PartialState {}
339
340    fn restore_partial(&mut self, _: Self::PartialState) {}
341
342    fn is_partial_supported() -> bool {
343        false
344    }
345}
346
347impl<'a> winnow::stream::FindSlice<&str> for TokenSlice<'a> {
348    fn find_slice(&self, substr: &str) -> Option<std::ops::Range<usize>> {
349        self.iter()
350            .enumerate()
351            .find_map(|(i, b)| if b.value == substr { Some(i..self.end) } else { None })
352    }
353}
354
355#[derive(Clone, Debug)]
356pub struct Checkpoint(usize, usize);
357
358/// The types of tokens.
359#[derive(Debug, PartialEq, Eq, Copy, Clone, Display)]
360#[display(style = "camelCase")]
361pub enum TokenType {
362    /// A number.
363    Number,
364    /// A word.
365    Word,
366    /// An operator.
367    Operator,
368    /// A string.
369    String,
370    /// A keyword.
371    Keyword,
372    /// A type.
373    Type,
374    /// A brace.
375    Brace,
376    /// A hash.
377    Hash,
378    /// A bang.
379    Bang,
380    /// A dollar sign.
381    Dollar,
382    /// Whitespace.
383    Whitespace,
384    /// A comma.
385    Comma,
386    /// A colon.
387    Colon,
388    /// A double colon: `::`
389    DoubleColon,
390    /// A period.
391    Period,
392    /// A double period: `..`.
393    DoublePeriod,
394    /// A double period and a less than: `..<`.
395    DoublePeriodLessThan,
396    /// A line comment.
397    LineComment,
398    /// A block comment.
399    BlockComment,
400    /// A function name.
401    Function,
402    /// Unknown lexemes.
403    Unknown,
404    /// The ? symbol, used for optional values.
405    QuestionMark,
406    /// The @ symbol.
407    At,
408    /// `;`
409    SemiColon,
410}
411
412/// Most KCL tokens correspond to LSP semantic tokens (but not all).
413impl TryFrom<TokenType> for SemanticTokenType {
414    type Error = anyhow::Error;
415    fn try_from(token_type: TokenType) -> Result<Self> {
416        // If you return a new kind of `SemanticTokenType`, make sure to update `SEMANTIC_TOKEN_TYPES`
417        // in the LSP implementation.
418        Ok(match token_type {
419            TokenType::Number => Self::NUMBER,
420            TokenType::Word => Self::VARIABLE,
421            TokenType::Keyword => Self::KEYWORD,
422            TokenType::Type => Self::TYPE,
423            TokenType::Operator => Self::OPERATOR,
424            TokenType::QuestionMark => Self::OPERATOR,
425            TokenType::String => Self::STRING,
426            TokenType::Bang => Self::OPERATOR,
427            TokenType::LineComment => Self::COMMENT,
428            TokenType::BlockComment => Self::COMMENT,
429            TokenType::Function => Self::FUNCTION,
430            TokenType::Whitespace
431            | TokenType::Brace
432            | TokenType::Comma
433            | TokenType::Colon
434            | TokenType::DoubleColon
435            | TokenType::Period
436            | TokenType::DoublePeriod
437            | TokenType::DoublePeriodLessThan
438            | TokenType::Hash
439            | TokenType::Dollar
440            | TokenType::At
441            | TokenType::SemiColon
442            | TokenType::Unknown => {
443                anyhow::bail!("unsupported token type: {:?}", token_type)
444            }
445        })
446    }
447}
448
449impl TokenType {
450    pub fn is_whitespace(&self) -> bool {
451        matches!(self, Self::Whitespace)
452    }
453
454    pub fn is_comment(&self) -> bool {
455        matches!(self, Self::LineComment | Self::BlockComment)
456    }
457}
458
459#[derive(Debug, PartialEq, Eq, Clone)]
460pub struct Token {
461    pub token_type: TokenType,
462    /// Offset in the source code where this token begins.
463    pub start: usize,
464    /// Offset in the source code where this token ends.
465    pub end: usize,
466    pub(super) module_id: ModuleId,
467    pub(super) value: String,
468}
469
470impl ContainsToken<Token> for (TokenType, &str) {
471    fn contains_token(&self, token: Token) -> bool {
472        self.0 == token.token_type && self.1 == token.value
473    }
474}
475
476impl ContainsToken<Token> for TokenType {
477    fn contains_token(&self, token: Token) -> bool {
478        *self == token.token_type
479    }
480}
481
482impl Token {
483    pub fn from_range(
484        range: std::ops::Range<usize>,
485        module_id: ModuleId,
486        token_type: TokenType,
487        value: String,
488    ) -> Self {
489        Self {
490            start: range.start,
491            end: range.end,
492            module_id,
493            value,
494            token_type,
495        }
496    }
497    pub fn is_code_token(&self) -> bool {
498        !matches!(
499            self.token_type,
500            TokenType::Whitespace | TokenType::LineComment | TokenType::BlockComment
501        )
502    }
503
504    pub fn as_source_range(&self) -> SourceRange {
505        SourceRange::new(self.start, self.end, self.module_id)
506    }
507
508    pub fn as_source_ranges(&self) -> Vec<SourceRange> {
509        vec![self.as_source_range()]
510    }
511
512    pub fn visibility_keyword(&self) -> Option<ItemVisibility> {
513        if !matches!(self.token_type, TokenType::Keyword) {
514            return None;
515        }
516        match self.value.as_str() {
517            "export" => Some(ItemVisibility::Export),
518            _ => None,
519        }
520    }
521
522    pub fn numeric_value(&self) -> Option<f64> {
523        if self.token_type != TokenType::Number {
524            return None;
525        }
526        let value = &self.value;
527        let value = value
528            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
529            .map(|(s, _)| s)
530            .unwrap_or(value);
531        value.parse().ok()
532    }
533
534    pub fn uint_value(&self) -> Option<u32> {
535        if self.token_type != TokenType::Number {
536            return None;
537        }
538        let value = &self.value;
539        let value = value
540            .split_once(|c: char| c == '_' || c.is_ascii_alphabetic())
541            .map(|(s, _)| s)
542            .unwrap_or(value);
543        value.parse().ok()
544    }
545
546    pub fn numeric_suffix(&self) -> NumericSuffix {
547        if self.token_type != TokenType::Number {
548            return NumericSuffix::None;
549        }
550
551        if self.value.ends_with('_') {
552            return NumericSuffix::Count;
553        }
554
555        for suffix in NUM_SUFFIXES {
556            if self.value.ends_with(suffix) {
557                return suffix.parse().unwrap();
558            }
559        }
560
561        NumericSuffix::None
562    }
563
564    /// Is this token the beginning of a variable/function declaration?
565    /// If so, what kind?
566    /// If not, returns None.
567    pub fn declaration_keyword(&self) -> Option<VariableKind> {
568        if !matches!(self.token_type, TokenType::Keyword) {
569            return None;
570        }
571        Some(match self.value.as_str() {
572            "fn" => VariableKind::Fn,
573            "var" | "let" | "const" => VariableKind::Const,
574            _ => return None,
575        })
576    }
577}
578
579impl From<Token> for SourceRange {
580    fn from(token: Token) -> Self {
581        Self::new(token.start, token.end, token.module_id)
582    }
583}
584
585impl From<&Token> for SourceRange {
586    fn from(token: &Token) -> Self {
587        Self::new(token.start, token.end, token.module_id)
588    }
589}
590
591pub fn lex(s: &str, module_id: ModuleId) -> Result<TokenStream, KclError> {
592    tokeniser::lex(s, module_id).map_err(From::from)
593}
594
595impl From<ParseError<Input<'_>, winnow::error::ContextError>> for KclError {
596    fn from(err: ParseError<Input<'_>, winnow::error::ContextError>) -> Self {
597        let (input, offset): (Vec<char>, usize) = (err.input().chars().collect(), err.offset());
598        let module_id = err.input().state.module_id;
599
600        if offset >= input.len() {
601            // From the winnow docs:
602            //
603            // This is an offset, not an index, and may point to
604            // the end of input (input.len()) on eof errors.
605
606            return KclError::new_lexical(crate::errors::KclErrorDetails::new(
607                "unexpected EOF while parsing".to_owned(),
608                vec![SourceRange::new(offset, offset, module_id)],
609            ));
610        }
611
612        // TODO: Add the Winnow tokenizer context to the error.
613        // See https://github.com/KittyCAD/modeling-app/issues/784
614        let bad_token = &input[offset];
615        // TODO: Add the Winnow parser context to the error.
616        // See https://github.com/KittyCAD/modeling-app/issues/784
617        KclError::new_lexical(crate::errors::KclErrorDetails::new(
618            format!("found unknown token '{bad_token}'"),
619            vec![SourceRange::new(offset, offset + 1, module_id)],
620        ))
621    }
622}
kcl_lib/parsing/token/mod.rs

kcl_lib/parsing/token/
mod.rs
