wac_parser/
lexer.rs

1//! Module for the lexer implementation.
2
3use logos::{Logos, SpannedIter};
4use miette::SourceSpan;
5use std::fmt;
6
7fn to_source_span(span: logos::Span) -> SourceSpan {
8    SourceSpan::new(span.start.into(), span.end - span.start)
9}
10
11/// Represents a lexer error.
12#[derive(thiserror::Error, Debug, Clone, Copy, PartialEq, Eq, Default)]
13pub enum Error {
14    /// An unexpected token was encountered.
15    #[default]
16    #[error("unexpected token was encountered")]
17    UnexpectedToken,
18    /// An unterminated string was encountered.
19    #[error("an unterminated string was encountered")]
20    UnterminatedString,
21    /// An unterminated comment was encountered.
22    #[error("an unterminated comment was encountered")]
23    UnterminatedComment,
24    /// A disallowed bidirectional override codepoint was encountered.
25    #[error("disallowed bidirectional override codepoint `{c}`", c = .0.escape_unicode())]
26    DisallowedBidirectionalOverride(char),
27    /// A discouraged Unicode codepoint was encountered.
28    #[error("codepoint `{c}` is discouraged by Unicode", c = .0.escape_unicode())]
29    DiscouragedUnicodeCodepoint(char),
30    /// A disallowed control code was encountered.
31    #[error("disallowed control code '{c}'", c = .0.escape_unicode())]
32    DisallowedControlCode(char),
33}
34
35impl From<()> for Error {
36    fn from(_: ()) -> Self {
37        Error::UnexpectedToken
38    }
39}
40
41fn detect_invalid_input(source: &str) -> Result<(), (Error, SourceSpan)> {
42    for (offset, ch) in source.char_indices() {
43        match ch {
44            '\r' | '\t' | '\n' => {}
45
46            // Bidirectional override codepoints can be used to craft source code that
47            // appears to have a different meaning than its actual meaning. See
48            // [CVE-2021-42574] for background and motivation.
49            //
50            // [CVE-2021-42574]: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-42574
51            '\u{202a}' | '\u{202b}' | '\u{202c}' | '\u{202d}' | '\u{202e}' | '\u{2066}'
52            | '\u{2067}' | '\u{2068}' | '\u{2069}' => {
53                return Err((
54                    Error::DisallowedBidirectionalOverride(ch),
55                    SourceSpan::new(offset.into(), ch.len_utf8()),
56                ));
57            }
58
59            // Disallow several characters which are deprecated or discouraged in Unicode.
60            //
61            // U+149 deprecated; see Unicode 13.0.0, sec. 7.1 Latin, Compatibility Digraphs.
62            // U+673 deprecated; see Unicode 13.0.0, sec. 9.2 Arabic, Additional Vowel Marks.
63            // U+F77 and U+F79 deprecated; see Unicode 13.0.0, sec. 13.4 Tibetan, Vowels.
64            // U+17A3 and U+17A4 deprecated, and U+17B4 and U+17B5 discouraged; see
65            // Unicode 13.0.0, sec. 16.4 Khmer, Characters Whose Use Is Discouraged.
66            '\u{149}' | '\u{673}' | '\u{f77}' | '\u{f79}' | '\u{17a3}' | '\u{17a4}'
67            | '\u{17b4}' | '\u{17b5}' => {
68                return Err((
69                    Error::DiscouragedUnicodeCodepoint(ch),
70                    SourceSpan::new(offset.into(), ch.len_utf8()),
71                ));
72            }
73
74            // Disallow control codes other than the ones explicitly recognized above,
75            // so that viewing a wit file on a terminal doesn't have surprising side
76            // effects or appear to have a different meaning than its actual meaning.
77            ch if ch.is_control() => {
78                return Err((
79                    Error::DisallowedControlCode(ch),
80                    SourceSpan::new(offset.into(), ch.len_utf8()),
81                ));
82            }
83
84            _ => {}
85        }
86    }
87
88    Ok(())
89}
90
91/// Represents a WAC token.
92#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
93#[logos(error = Error)]
94#[logos(skip r"[ \t\r\n\f]+")]
95#[logos(subpattern word = r"[a-z][a-z0-9]*|[A-Z][A-Z0-9]*")]
96#[logos(subpattern id = r"%?(?&word)(-(?&word))*")]
97#[logos(subpattern package_name = r"(?&id)(:(?&id))+")]
98#[logos(subpattern semver = r"([0-9]+)(\.[0-9a-zA-Z-\+]+)*")]
99pub enum Token {
100    /// A comment.
101    #[regex(r"//[^\n]*", logos::skip)]
102    Comment,
103
104    /// A block comment.
105    #[token(r"/*", helpers::skip_block_comment)]
106    BlockComment,
107
108    /// An identifier.
109    #[regex(r"(?&id)")]
110    Ident,
111
112    /// A string literal.
113    #[token("\"", helpers::string)]
114    String,
115
116    /// A package name.
117    #[regex(r"(?&package_name)(@(?&semver))?")]
118    PackageName,
119
120    /// A package path with optional semantic version.
121    #[regex(r"(?&package_name)(/(?&id))+(@(?&semver))?")]
122    PackagePath,
123
124    /// The `import` keyword.
125    #[token("import")]
126    ImportKeyword,
127    /// The `with` keyword.
128    #[token("with")]
129    WithKeyword,
130    /// The `type` keyword.
131    #[token("type")]
132    TypeKeyword,
133    /// The `tuple` keyword.
134    #[token("tuple")]
135    TupleKeyword,
136    /// The `list` keyword.
137    #[token("list")]
138    ListKeyword,
139    /// The `option` keyword.
140    #[token("option")]
141    OptionKeyword,
142    /// The `result` keyword.
143    #[token("result")]
144    ResultKeyword,
145    /// The `borrow` keyword.
146    #[token("borrow")]
147    BorrowKeyword,
148    /// The `resource` keyword.
149    #[token("resource")]
150    ResourceKeyword,
151    /// The `variant` keyword.
152    #[token("variant")]
153    VariantKeyword,
154    /// The `record` keyword.
155    #[token("record")]
156    RecordKeyword,
157    /// The `flags` keyword.
158    #[token("flags")]
159    FlagsKeyword,
160    /// The `enum` keyword.
161    #[token("enum")]
162    EnumKeyword,
163    /// The `func` keyword.
164    #[token("func")]
165    FuncKeyword,
166    /// The `static` keyword.
167    #[token("static")]
168    StaticKeyword,
169    /// The `constructor` keyword.
170    #[token("constructor")]
171    ConstructorKeyword,
172    /// The `u8` keyword.
173    #[token("u8")]
174    U8Keyword,
175    /// The `s8` keyword.
176    #[token("s8")]
177    S8Keyword,
178    /// The `u16` keyword.
179    #[token("u16")]
180    U16Keyword,
181    /// The `s16` keyword.
182    #[token("s16")]
183    S16Keyword,
184    /// The `u32` keyword.
185    #[token("u32")]
186    U32Keyword,
187    /// The `s32` keyword.
188    #[token("s32")]
189    S32Keyword,
190    /// The `u64` keyword.
191    #[token("u64")]
192    U64Keyword,
193    /// The `s64` keyword.
194    #[token("s64")]
195    S64Keyword,
196    /// The `f32` keyword.
197    #[token("f32")]
198    F32Keyword,
199    /// The `f64` keyword.
200    #[token("f64")]
201    F64Keyword,
202    /// The `char` keyword.
203    #[token("char")]
204    CharKeyword,
205    /// The `bool` keyword.
206    #[token("bool")]
207    BoolKeyword,
208    /// The `string` keyword.
209    #[token("string")]
210    StringKeyword,
211    /// The `interface` keyword.
212    #[token("interface")]
213    InterfaceKeyword,
214    /// The `world` keyword.
215    #[token("world")]
216    WorldKeyword,
217    /// The `export` keyword.
218    #[token("export")]
219    ExportKeyword,
220    /// The `new` keyword.
221    #[token("new")]
222    NewKeyword,
223    /// The `let` keyword.
224    #[token("let")]
225    LetKeyword,
226    /// The `use` keyword.
227    #[token("use")]
228    UseKeyword,
229    /// The `include` keyword.
230    #[token("include")]
231    IncludeKeyword,
232    /// The `as` keyword.
233    #[token("as")]
234    AsKeyword,
235    /// The `package` keyword.
236    #[token("package")]
237    PackageKeyword,
238    /// The `targets` keyword.
239    #[token("targets")]
240    TargetsKeyword,
241    /// The `async` keyword.
242    #[token("async")]
243    AsyncKeyword,
244    /// The `stream` keyword.
245    #[token("stream")]
246    StreamKeyword,
247    /// The `future` keyword.
248    #[token("future")]
249    FutureKeyword,
250    /// The `error-context` keyword.
251    #[token("error-context")]
252    ErrorContextKeyword,
253
254    /// The `;` symbol.
255    #[token(";")]
256    Semicolon,
257    /// The `{` symbol.
258    #[token("{")]
259    OpenBrace,
260    /// The `}` symbol.
261    #[token("}")]
262    CloseBrace,
263    /// The `:` symbol.
264    #[token(":")]
265    Colon,
266    /// The `=` symbol.
267    #[token("=")]
268    Equals,
269    /// The `(` symbol.
270    #[token("(")]
271    OpenParen,
272    /// The `)` symbol.
273    #[token(")")]
274    CloseParen,
275    /// The `->` symbol.
276    #[token("->")]
277    Arrow,
278    /// The `<` symbol.
279    #[token("<")]
280    OpenAngle,
281    /// The `>` symbol.
282    #[token(">")]
283    CloseAngle,
284    /// The `_` symbol.
285    #[token("_")]
286    Underscore,
287    /// The `[` symbol.
288    #[token("[")]
289    OpenBracket,
290    /// The `]` symbol.
291    #[token("]")]
292    CloseBracket,
293    /// The `.` symbol.
294    #[token(".")]
295    Dot,
296    /// The `...` symbol.
297    #[token("...")]
298    Ellipsis,
299    /// The `,` symbol.
300    #[token(",")]
301    Comma,
302    /// The `/` symbol.
303    #[token("/")]
304    Slash,
305    /// The `@` symbol.
306    #[token("@")]
307    At,
308}
309
310impl fmt::Display for Token {
311    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
312        match self {
313            Token::Comment | Token::BlockComment => write!(f, "comment"),
314            Token::Ident => write!(f, "identifier"),
315            Token::String => write!(f, "string literal"),
316            Token::PackageName => write!(f, "package name"),
317            Token::PackagePath => write!(f, "package path"),
318            Token::ImportKeyword => write!(f, "`import` keyword"),
319            Token::WithKeyword => write!(f, "`with` keyword"),
320            Token::TypeKeyword => write!(f, "`type` keyword"),
321            Token::TupleKeyword => write!(f, "`tuple` keyword"),
322            Token::ListKeyword => write!(f, "`list` keyword"),
323            Token::OptionKeyword => write!(f, "`option` keyword"),
324            Token::ResultKeyword => write!(f, "`result` keyword"),
325            Token::BorrowKeyword => write!(f, "`borrow` keyword"),
326            Token::ResourceKeyword => write!(f, "`resource` keyword"),
327            Token::VariantKeyword => write!(f, "`variant` keyword"),
328            Token::RecordKeyword => write!(f, "`record` keyword"),
329            Token::FlagsKeyword => write!(f, "`flags` keyword"),
330            Token::EnumKeyword => write!(f, "`enum` keyword"),
331            Token::FuncKeyword => write!(f, "`func` keyword"),
332            Token::StaticKeyword => write!(f, "`static` keyword"),
333            Token::ConstructorKeyword => write!(f, "`constructor` keyword"),
334            Token::U8Keyword => write!(f, "`u8` keyword"),
335            Token::S8Keyword => write!(f, "`s8` keyword"),
336            Token::U16Keyword => write!(f, "`u16` keyword"),
337            Token::S16Keyword => write!(f, "`s16` keyword"),
338            Token::U32Keyword => write!(f, "`u32` keyword"),
339            Token::S32Keyword => write!(f, "`s32` keyword"),
340            Token::U64Keyword => write!(f, "`u64` keyword"),
341            Token::S64Keyword => write!(f, "`s64` keyword"),
342            Token::F32Keyword => write!(f, "`f32` keyword"),
343            Token::F64Keyword => write!(f, "`f64` keyword"),
344            Token::CharKeyword => write!(f, "`char` keyword"),
345            Token::BoolKeyword => write!(f, "`bool` keyword"),
346            Token::StringKeyword => write!(f, "`string` keyword"),
347            Token::InterfaceKeyword => write!(f, "`interface` keyword"),
348            Token::WorldKeyword => write!(f, "`world` keyword"),
349            Token::ExportKeyword => write!(f, "`export` keyword"),
350            Token::NewKeyword => write!(f, "`new` keyword"),
351            Token::LetKeyword => write!(f, "`let` keyword"),
352            Token::UseKeyword => write!(f, "`use` keyword"),
353            Token::IncludeKeyword => write!(f, "`include` keyword"),
354            Token::AsKeyword => write!(f, "`as` keyword"),
355            Token::PackageKeyword => write!(f, "`package` keyword"),
356            Token::TargetsKeyword => write!(f, "`targets` keyword"),
357            Token::AsyncKeyword => write!(f, "`async` keyword"),
358            Token::StreamKeyword => write!(f, "`stream` keyword"),
359            Token::FutureKeyword => write!(f, "`future` keyword"),
360            Token::ErrorContextKeyword => write!(f, "`error-context` keyword"),
361            Token::Semicolon => write!(f, "`;`"),
362            Token::OpenBrace => write!(f, "`{{`"),
363            Token::CloseBrace => write!(f, "`}}`"),
364            Token::Colon => write!(f, "`:`"),
365            Token::Equals => write!(f, "`=`"),
366            Token::OpenParen => write!(f, "`(`"),
367            Token::CloseParen => write!(f, "`)`"),
368            Token::Arrow => write!(f, "`->`"),
369            Token::OpenAngle => write!(f, "`<`"),
370            Token::CloseAngle => write!(f, "`>`"),
371            Token::Underscore => write!(f, "`_`"),
372            Token::OpenBracket => write!(f, "`[`"),
373            Token::CloseBracket => write!(f, "`]`"),
374            Token::Dot => write!(f, "`.`"),
375            Token::Ellipsis => write!(f, "`...`"),
376            Token::Comma => write!(f, "`,`"),
377            Token::Slash => write!(f, "`/`"),
378            Token::At => write!(f, "`@`"),
379        }
380    }
381}
382
383mod helpers {
384    use super::{Error, Token};
385    use logos::{FilterResult, Logos};
386
387    /// Represents a WAC comment token.
388    #[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
389    #[logos(error = Error)]
390    #[logos(skip r"[ \t\n\f]+")]
391    pub enum CommentToken<'a> {
392        /// A comment.
393        #[regex(r"//[^\n]*")]
394        Comment(&'a str),
395
396        /// A block comment.
397        #[token(r"/*", block_comment)]
398        BlockComment(&'a str),
399    }
400
401    pub fn string(lex: &mut logos::Lexer<Token>) -> Result<(), Error> {
402        let remainder = lex.remainder();
403        let len = remainder.find('"').ok_or(Error::UnterminatedString)?;
404        lex.bump(len + 1 /* opening quote */);
405        Ok(())
406    }
407
408    pub fn block_comment_length(bytes: &[u8]) -> Option<usize> {
409        let mut iter = bytes.iter().copied().peekable();
410        let mut depth = 1;
411        let mut len = 0;
412        while depth > 0 {
413            len += 1;
414            match iter.next()? {
415                b'/' if iter.peek() == Some(&b'*') => {
416                    depth += 1;
417                    len += 1;
418                    iter.next();
419                }
420                b'*' if iter.peek() == Some(&b'/') => {
421                    depth -= 1;
422                    len += 1;
423                    iter.next();
424                }
425                _ => {}
426            }
427        }
428
429        Some(len + 2 /* opening tokens */)
430    }
431
432    pub fn block_comment<'a>(
433        lex: &mut logos::Lexer<'a, CommentToken<'a>>,
434    ) -> Result<&'a str, Error> {
435        let span = lex.span();
436        match block_comment_length(lex.remainder().as_bytes()) {
437            Some(len) => {
438                let s = &lex.source()[span.start..span.start + len];
439                lex.bump(len - 2 /* opening tokens */);
440                Ok(s)
441            }
442            None => {
443                lex.bump(lex.remainder().len());
444                Err(Error::UnterminatedComment)
445            }
446        }
447    }
448
449    pub fn skip_block_comment(lex: &mut logos::Lexer<Token>) -> FilterResult<(), Error> {
450        match block_comment_length(lex.remainder().as_bytes()) {
451            Some(len) => {
452                lex.bump(len - 2 /* opening tokens */);
453                FilterResult::Skip
454            }
455            None => {
456                lex.bump(lex.remainder().len());
457                FilterResult::Error(Error::UnterminatedComment)
458            }
459        }
460    }
461}
462
463/// The result type for the lexer.
464pub type LexerResult<T> = Result<T, Error>;
465
466/// Implements a WAC lexer.
467pub struct Lexer<'a>(SpannedIter<'a, Token>);
468
469impl<'a> Lexer<'a> {
470    /// Creates a new lexer for the given source string.
471    pub fn new(source: &'a str) -> Result<Self, (Error, SourceSpan)> {
472        detect_invalid_input(source)?;
473        Ok(Self(Token::lexer(source).spanned()))
474    }
475
476    /// Gets the source string of the given span.
477    pub fn source(&self, span: SourceSpan) -> &'a str {
478        &self.0.source()[span.offset()..span.offset() + span.len()]
479    }
480
481    /// Gets the current span of the lexer.
482    pub fn span(&self) -> SourceSpan {
483        let mut span = self.0.span();
484        if span.end == self.0.source().len() {
485            // Currently miette silently fails to display a label
486            // if the span is at the end of the source; this means
487            // we can't properly show the "end of input" span.
488            // For now, have the span point at the last byte in the source.
489            // See: https://github.com/zkat/miette/issues/219
490            span.start = span.start.saturating_sub(1);
491            span.end = span.start + 1;
492        }
493
494        to_source_span(span)
495    }
496
497    /// Peeks at the next token.
498    pub fn peek(&self) -> Option<(LexerResult<Token>, SourceSpan)> {
499        let mut lexer = self.0.clone();
500        lexer.next().map(|(r, s)| (r, to_source_span(s)))
501    }
502
503    /// Peeks at the token after the next token.
504    pub fn peek2(&self) -> Option<(LexerResult<Token>, SourceSpan)> {
505        let mut lexer = self.0.clone();
506        lexer.next();
507        lexer.next().map(|(r, s)| (r, to_source_span(s)))
508    }
509
510    /// Consumes available documentation comment tokens.
511    pub fn comments(&self) -> Result<Vec<(&'a str, SourceSpan)>, (Error, SourceSpan)> {
512        let mut comments = Vec::new();
513        let mut lexer = (*self.0).clone().morph::<helpers::CommentToken>().spanned();
514        while let Some((Ok(token), span)) = lexer.next() {
515            match token {
516                helpers::CommentToken::Comment(c) | helpers::CommentToken::BlockComment(c) => {
517                    let c = if let Some(c) = c.strip_prefix("///") {
518                        c.trim()
519                    } else if let Some(c) = c.strip_prefix("/**") {
520                        if c == "/" {
521                            continue;
522                        }
523                        c.strip_suffix("*/").unwrap().trim()
524                    } else {
525                        continue;
526                    };
527                    comments.push((c, to_source_span(span)));
528                }
529            }
530        }
531        Ok(comments)
532    }
533}
534
535impl Iterator for Lexer<'_> {
536    type Item = (LexerResult<Token>, SourceSpan);
537
538    fn next(&mut self) -> Option<Self::Item> {
539        self.0.next().map(|(r, s)| (r, to_source_span(s)))
540    }
541}
542
543#[cfg(test)]
544mod test {
545    use super::*;
546    use logos::{Logos, Source};
547    use std::{fmt, ops::Range};
548
549    //use super::*;
550
551    #[allow(clippy::type_complexity)]
552    pub fn assert_lex<'a, Token>(
553        source: &'a Token::Source,
554        tokens: &[(
555            Result<Token, Token::Error>,
556            <Token::Source as Source>::Slice<'a>,
557            Range<usize>,
558        )],
559    ) where
560        Token: Logos<'a> + fmt::Debug + PartialEq,
561        Token::Extras: Default,
562    {
563        let mut lex = Token::lexer(source);
564
565        for tuple in tokens {
566            assert_eq!(
567                &(lex.next().expect("unexpected end"), lex.slice(), lex.span()),
568                tuple
569            );
570        }
571
572        assert_eq!(lex.next(), None, "tokens remain");
573    }
574
575    #[test]
576    fn comments() {
577        assert_lex::<Token>(
578            r#"
579            //
580            // comment
581            /**/
582            /* a block comment */
583            /* a multi
584               line comment
585             */
586            /* a /* /* deeply */ nested */ block comment */
587            "#,
588            &[],
589        );
590    }
591
592    #[test]
593    fn unterminated_comment() {
594        let source = r#"/* /* unterminated */"#;
595
596        assert_lex::<Token>(
597            source,
598            &[(
599                Err(Error::UnterminatedComment),
600                "/* /* unterminated */",
601                0..21,
602            )],
603        );
604    }
605
606    #[test]
607    fn ident() {
608        assert_lex(
609            r#"
610            foo
611            foo123
612            f-b
613            foo-bar123
614            foo0123-bar0123-baz0123
615            %interface
616            foo123-BAR
617            "#,
618            &[
619                (Ok(Token::Ident), "foo", 13..16),
620                (Ok(Token::Ident), "foo123", 29..35),
621                (Ok(Token::Ident), "f-b", 48..51),
622                (Ok(Token::Ident), "foo-bar123", 64..74),
623                (Ok(Token::Ident), "foo0123-bar0123-baz0123", 87..110),
624                (Ok(Token::Ident), "%interface", 123..133),
625                (Ok(Token::Ident), "foo123-BAR", 146..156),
626            ],
627        );
628    }
629
630    #[test]
631    fn string() {
632        assert_lex(
633            r#"
634            ""
635            "foo"
636            "foo  bar"
637            "foo
638            bar"
639            "#,
640            &[
641                (Ok(Token::String), "\"\"", 13..15),
642                (Ok(Token::String), "\"foo\"", 28..33),
643                (Ok(Token::String), "\"foo  bar\"", 46..56),
644                (Ok(Token::String), "\"foo\n            bar\"", 69..90),
645            ],
646        );
647    }
648
649    #[test]
650    fn package_path() {
651        assert_lex(
652            r#"
653foo:bar/baz/qux/jam
654foo:bar:baz:qux/jam
655foo:bar/baz@0.0.4
656foo:bar/baz@1.2.3
657foo:bar/baz@10.20.30
658foo:bar/baz@1.1.2-prerelease+meta
659foo:bar/baz@1.1.2+meta
660foo:bar/baz@1.1.2+meta-valid
661foo:bar/baz@1.0.0-alpha
662foo:bar/baz@1.0.0-beta
663foo:bar/baz@1.0.0-alpha.beta
664foo:bar/baz@1.0.0-alpha.beta.1
665foo:bar/baz@1.0.0-alpha.1
666foo:bar/baz@1.0.0-alpha0.valid
667foo:bar/baz@1.0.0-alpha.0valid
668foo:bar/baz@1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay
669foo:bar/baz@1.0.0-rc.1+build.1
670foo:bar/baz@2.0.0-rc.1+build.123
671foo:bar/baz@1.2.3-beta
672foo:bar/baz@10.2.3-DEV-SNAPSHOT
673foo:bar/baz@1.2.3-SNAPSHOT-123
674foo:bar/baz@1.0.0
675foo:bar/baz@2.0.0
676foo:bar/baz@1.1.7
677foo:bar/baz@2.0.0+build.1848
678foo:bar/baz@2.0.1-alpha.1227
679foo:bar/baz@1.0.0-alpha+beta
680foo:bar/baz@1.2.3----RC-SNAPSHOT.12.9.1--.12+788
681foo:bar/baz@1.2.3----R-S.12.9.1--.12+meta
682foo:bar/baz@1.2.3----RC-SNAPSHOT.12.9.1--.12
683foo:bar/baz@1.0.0+0.build.1-rc.10000aaa-kk-0.1
684foo:bar/baz@99999999999999999999999.999999999999999999.99999999999999999
685foo:bar/baz@1.0.0-0A.is.legal
686"#,
687            &[
688                (Ok(Token::PackagePath), "foo:bar/baz/qux/jam", 1..20),
689                (Ok(Token::PackagePath), "foo:bar:baz:qux/jam", 21..40),
690                (Ok(Token::PackagePath), "foo:bar/baz@0.0.4", 41..58),
691                (Ok(Token::PackagePath), "foo:bar/baz@1.2.3", 59..76),
692                (Ok(Token::PackagePath), "foo:bar/baz@10.20.30", 77..97),
693                (
694                    Ok(Token::PackagePath),
695                    "foo:bar/baz@1.1.2-prerelease+meta",
696                    98..131,
697                ),
698                (Ok(Token::PackagePath), "foo:bar/baz@1.1.2+meta", 132..154),
699                (
700                    Ok(Token::PackagePath),
701                    "foo:bar/baz@1.1.2+meta-valid",
702                    155..183,
703                ),
704                (Ok(Token::PackagePath), "foo:bar/baz@1.0.0-alpha", 184..207),
705                (Ok(Token::PackagePath), "foo:bar/baz@1.0.0-beta", 208..230),
706                (
707                    Ok(Token::PackagePath),
708                    "foo:bar/baz@1.0.0-alpha.beta",
709                    231..259,
710                ),
711                (
712                    Ok(Token::PackagePath),
713                    "foo:bar/baz@1.0.0-alpha.beta.1",
714                    260..290,
715                ),
716                (
717                    Ok(Token::PackagePath),
718                    "foo:bar/baz@1.0.0-alpha.1",
719                    291..316,
720                ),
721                (
722                    Ok(Token::PackagePath),
723                    "foo:bar/baz@1.0.0-alpha0.valid",
724                    317..347,
725                ),
726                (
727                    Ok(Token::PackagePath),
728                    "foo:bar/baz@1.0.0-alpha.0valid",
729                    348..378,
730                ),
731                (
732                    Ok(Token::PackagePath),
733                    "foo:bar/baz@1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay",
734                    379..445,
735                ),
736                (
737                    Ok(Token::PackagePath),
738                    "foo:bar/baz@1.0.0-rc.1+build.1",
739                    446..476,
740                ),
741                (
742                    Ok(Token::PackagePath),
743                    "foo:bar/baz@2.0.0-rc.1+build.123",
744                    477..509,
745                ),
746                (Ok(Token::PackagePath), "foo:bar/baz@1.2.3-beta", 510..532),
747                (
748                    Ok(Token::PackagePath),
749                    "foo:bar/baz@10.2.3-DEV-SNAPSHOT",
750                    533..564,
751                ),
752                (
753                    Ok(Token::PackagePath),
754                    "foo:bar/baz@1.2.3-SNAPSHOT-123",
755                    565..595,
756                ),
757                (Ok(Token::PackagePath), "foo:bar/baz@1.0.0", 596..613),
758                (Ok(Token::PackagePath), "foo:bar/baz@2.0.0", 614..631),
759                (Ok(Token::PackagePath), "foo:bar/baz@1.1.7", 632..649),
760                (
761                    Ok(Token::PackagePath),
762                    "foo:bar/baz@2.0.0+build.1848",
763                    650..678,
764                ),
765                (
766                    Ok(Token::PackagePath),
767                    "foo:bar/baz@2.0.1-alpha.1227",
768                    679..707,
769                ),
770                (
771                    Ok(Token::PackagePath),
772                    "foo:bar/baz@1.0.0-alpha+beta",
773                    708..736,
774                ),
775                (
776                    Ok(Token::PackagePath),
777                    "foo:bar/baz@1.2.3----RC-SNAPSHOT.12.9.1--.12+788",
778                    737..785,
779                ),
780                (
781                    Ok(Token::PackagePath),
782                    "foo:bar/baz@1.2.3----R-S.12.9.1--.12+meta",
783                    786..827,
784                ),
785                (
786                    Ok(Token::PackagePath),
787                    "foo:bar/baz@1.2.3----RC-SNAPSHOT.12.9.1--.12",
788                    828..872,
789                ),
790                (
791                    Ok(Token::PackagePath),
792                    "foo:bar/baz@1.0.0+0.build.1-rc.10000aaa-kk-0.1",
793                    873..919,
794                ),
795                (
796                    Ok(Token::PackagePath),
797                    "foo:bar/baz@99999999999999999999999.999999999999999999.99999999999999999",
798                    920..992,
799                ),
800                (
801                    Ok(Token::PackagePath),
802                    "foo:bar/baz@1.0.0-0A.is.legal",
803                    993..1022,
804                ),
805            ],
806        );
807    }
808
809    #[test]
810    fn keywords() {
811        assert_lex(
812            r#"
813import
814with
815type
816tuple
817list
818option
819result
820borrow
821resource
822variant
823record
824flags
825enum
826func
827static
828constructor
829u8
830s8
831u16
832s16
833u32
834s32
835u64
836s64
837f32
838f64
839char
840bool
841string
842interface
843world
844export
845new
846let
847use
848include
849as
850package
851targets
852async
853stream
854future
855error-context
856            "#,
857            &[
858                (Ok(Token::ImportKeyword), "import", 1..7),
859                (Ok(Token::WithKeyword), "with", 8..12),
860                (Ok(Token::TypeKeyword), "type", 13..17),
861                (Ok(Token::TupleKeyword), "tuple", 18..23),
862                (Ok(Token::ListKeyword), "list", 24..28),
863                (Ok(Token::OptionKeyword), "option", 29..35),
864                (Ok(Token::ResultKeyword), "result", 36..42),
865                (Ok(Token::BorrowKeyword), "borrow", 43..49),
866                (Ok(Token::ResourceKeyword), "resource", 50..58),
867                (Ok(Token::VariantKeyword), "variant", 59..66),
868                (Ok(Token::RecordKeyword), "record", 67..73),
869                (Ok(Token::FlagsKeyword), "flags", 74..79),
870                (Ok(Token::EnumKeyword), "enum", 80..84),
871                (Ok(Token::FuncKeyword), "func", 85..89),
872                (Ok(Token::StaticKeyword), "static", 90..96),
873                (Ok(Token::ConstructorKeyword), "constructor", 97..108),
874                (Ok(Token::U8Keyword), "u8", 109..111),
875                (Ok(Token::S8Keyword), "s8", 112..114),
876                (Ok(Token::U16Keyword), "u16", 115..118),
877                (Ok(Token::S16Keyword), "s16", 119..122),
878                (Ok(Token::U32Keyword), "u32", 123..126),
879                (Ok(Token::S32Keyword), "s32", 127..130),
880                (Ok(Token::U64Keyword), "u64", 131..134),
881                (Ok(Token::S64Keyword), "s64", 135..138),
882                (Ok(Token::F32Keyword), "f32", 139..142),
883                (Ok(Token::F64Keyword), "f64", 143..146),
884                (Ok(Token::CharKeyword), "char", 147..151),
885                (Ok(Token::BoolKeyword), "bool", 152..156),
886                (Ok(Token::StringKeyword), "string", 157..163),
887                (Ok(Token::InterfaceKeyword), "interface", 164..173),
888                (Ok(Token::WorldKeyword), "world", 174..179),
889                (Ok(Token::ExportKeyword), "export", 180..186),
890                (Ok(Token::NewKeyword), "new", 187..190),
891                (Ok(Token::LetKeyword), "let", 191..194),
892                (Ok(Token::UseKeyword), "use", 195..198),
893                (Ok(Token::IncludeKeyword), "include", 199..206),
894                (Ok(Token::AsKeyword), "as", 207..209),
895                (Ok(Token::PackageKeyword), "package", 210..217),
896                (Ok(Token::TargetsKeyword), "targets", 218..225),
897                (Ok(Token::AsyncKeyword), "async", 226..231),
898                (Ok(Token::StreamKeyword), "stream", 232..238),
899                (Ok(Token::FutureKeyword), "future", 239..245),
900                (Ok(Token::ErrorContextKeyword), "error-context", 246..259),
901            ],
902        );
903    }
904
905    #[test]
906    fn symbols() {
907        assert_lex(
908            r#";{}:=()-><>_[]. ...,/@"#,
909            &[
910                (Ok(Token::Semicolon), ";", 0..1),
911                (Ok(Token::OpenBrace), "{", 1..2),
912                (Ok(Token::CloseBrace), "}", 2..3),
913                (Ok(Token::Colon), ":", 3..4),
914                (Ok(Token::Equals), "=", 4..5),
915                (Ok(Token::OpenParen), "(", 5..6),
916                (Ok(Token::CloseParen), ")", 6..7),
917                (Ok(Token::Arrow), "->", 7..9),
918                (Ok(Token::OpenAngle), "<", 9..10),
919                (Ok(Token::CloseAngle), ">", 10..11),
920                (Ok(Token::Underscore), "_", 11..12),
921                (Ok(Token::OpenBracket), "[", 12..13),
922                (Ok(Token::CloseBracket), "]", 13..14),
923                (Ok(Token::Dot), ".", 14..15),
924                (Ok(Token::Ellipsis), "...", 16..19),
925                (Ok(Token::Comma), ",", 19..20),
926                (Ok(Token::Slash), "/", 20..21),
927                (Ok(Token::At), "@", 21..22),
928            ],
929        );
930    }
931}
wac_parser/lexer.rs

wac_parser/
lexer.rs
