wit_parser/ast/
lex.rs

1use alloc::string::String;
2#[cfg(test)]
3use alloc::vec;
4use alloc::vec::Vec;
5use core::char;
6use core::fmt;
7use core::result::Result;
8use core::str;
9
10use self::Token::*;
11
12#[derive(Clone)]
13pub struct Tokenizer<'a> {
14    input: &'a str,
15    span_offset: u32,
16    chars: CrlfFold<'a>,
17}
18
19#[derive(Clone)]
20struct CrlfFold<'a> {
21    chars: str::CharIndices<'a>,
22}
23
24/// A span, designating a range of bytes where a token is located.
25///
26/// Uses `u32::MAX` as a sentinel value to represent unknown spans (e.g.,
27/// decoded from binary).
28#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)]
29pub struct Span {
30    start: u32,
31    end: u32,
32}
33
34impl Default for Span {
35    fn default() -> Span {
36        Span {
37            start: u32::MAX,
38            end: u32::MAX,
39        }
40    }
41}
42
43impl Span {
44    pub fn new(start: u32, end: u32) -> Span {
45        let span = Span { start, end };
46        assert!(span.is_known(), "cannot create a span with u32::MAX");
47        span
48    }
49
50    /// Adjusts this span by adding the given byte offset to both start and end.
51    pub fn adjust(&mut self, offset: u32) {
52        if self.is_known() {
53            self.start += offset;
54            self.end += offset;
55        }
56    }
57
58    /// Returns the start offset, panicking if this is an unknown span.
59    pub fn start(&self) -> u32 {
60        assert!(self.is_known(), "cannot get start of unknown span");
61        self.start
62    }
63
64    /// Returns the end offset, panicking if this is an unknown span.
65    pub fn end(&self) -> u32 {
66        assert!(self.is_known(), "cannot get end of unknown span");
67        self.end
68    }
69
70    /// Sets the end offset. If this is unknown, converts to a zero-width span at that position.
71    pub fn set_end(&mut self, new_end: u32) {
72        if !self.is_known() {
73            self.start = new_end;
74        }
75        self.end = new_end;
76    }
77
78    /// Sets the start offset. If this is unknown, converts to a zero-width span at that position.
79    pub fn set_start(&mut self, new_start: u32) {
80        if !self.is_known() {
81            self.end = new_start;
82        }
83        self.start = new_start;
84    }
85
86    /// Returns true if this span has a known source location.
87    pub fn is_known(&self) -> bool {
88        self.start != u32::MAX && self.end != u32::MAX
89    }
90}
91
92#[derive(Eq, PartialEq, Debug, Copy, Clone)]
93pub enum Token {
94    Whitespace,
95    Comment,
96
97    Equals,
98    Comma,
99    Colon,
100    Period,
101    Semicolon,
102    LeftParen,
103    RightParen,
104    LeftBrace,
105    RightBrace,
106    LessThan,
107    GreaterThan,
108    RArrow,
109    Star,
110    At,
111    Slash,
112    Plus,
113    Minus,
114    StringLiteral,
115
116    Use,
117    Type,
118    Func,
119    U8,
120    U16,
121    U32,
122    U64,
123    S8,
124    S16,
125    S32,
126    S64,
127    F32,
128    F64,
129    Char,
130    Record,
131    Resource,
132    Own,
133    Borrow,
134    Flags,
135    Variant,
136    Enum,
137    Bool,
138    String_,
139    Option_,
140    Result_,
141    Future,
142    Stream,
143    ErrorContext,
144    List,
145    Map,
146    Underscore,
147    As,
148    From_,
149    Static,
150    Interface,
151    Tuple,
152    Import,
153    Export,
154    World,
155    Package,
156    Constructor,
157    Async,
158
159    Id,
160    ExplicitId,
161
162    Integer,
163
164    Include,
165    With,
166}
167
168#[derive(Eq, PartialEq, Debug)]
169#[allow(dead_code)]
170pub enum Error {
171    ControlCodepoint(u32, char),
172    DeprecatedCodepoint(u32, char),
173    ForbiddenCodepoint(u32, char),
174    InvalidCharInId(u32, char),
175    IdPartEmpty(u32),
176    Unexpected(u32, char),
177    UnterminatedComment(u32),
178    Wanted {
179        at: u32,
180        expected: &'static str,
181        found: &'static str,
182    },
183    InvalidUnicodeValue(u32, u32),
184    InvalidStringElement(u32, char),
185    InvalidStringEscape(u32, char),
186    WantedChar(u32, char),
187    UnexpectedEof(u32),
188    InvalidUtf8(u32, core::str::Utf8Error),
189    NumberTooBig(u32),
190    LoneUnderscore(u32),
191    InvalidHexDigit(u32, char),
192}
193
194impl<'a> Tokenizer<'a> {
195    pub fn new(input: &'a str, span_offset: u32) -> Result<Tokenizer<'a>, Error> {
196        detect_invalid_input(input)?;
197
198        let mut t = Tokenizer {
199            input,
200            span_offset,
201            chars: CrlfFold {
202                chars: input.char_indices(),
203            },
204        };
205        // Eat utf-8 BOM
206        t.eatc('\u{feff}');
207        Ok(t)
208    }
209
210    pub fn expect_semicolon(&mut self) -> Result<(), Error> {
211        self.expect(Token::Semicolon)?;
212        Ok(())
213    }
214
215    pub fn get_span(&self, span: Span) -> &'a str {
216        let start = usize::try_from(span.start() - self.span_offset).unwrap();
217        let end = usize::try_from(span.end() - self.span_offset).unwrap();
218        &self.input[start..end]
219    }
220
221    pub fn parse_id(&self, span: Span) -> Result<&'a str, Error> {
222        let ret = self.get_span(span);
223        validate_id(span.start(), &ret)?;
224        Ok(ret)
225    }
226
227    pub fn parse_explicit_id(&self, span: Span) -> Result<&'a str, Error> {
228        let token = self.get_span(span);
229        let id_part = token.strip_prefix('%').unwrap();
230        validate_id(span.start(), id_part)?;
231        Ok(id_part)
232    }
233
234    pub fn next(&mut self) -> Result<Option<(Span, Token)>, Error> {
235        loop {
236            match self.next_raw()? {
237                Some((_, Token::Whitespace)) | Some((_, Token::Comment)) => {}
238                other => break Ok(other),
239            }
240        }
241    }
242
243    /// Three possibilities when calling this method: an `Err(...)` indicates that lexing failed, an
244    /// `Ok(Some(...))` produces the next token, and `Ok(None)` indicates that there are no more
245    /// tokens available.
246    pub fn next_raw(&mut self) -> Result<Option<(Span, Token)>, Error> {
247        let (str_start, ch) = match self.chars.next() {
248            Some(pair) => pair,
249            None => return Ok(None),
250        };
251        let start = self.span_offset + u32::try_from(str_start).unwrap();
252        let token = match ch {
253            '\n' | '\t' | ' ' => {
254                // Eat all contiguous whitespace tokens
255                while self.eatc(' ') || self.eatc('\t') || self.eatc('\n') {}
256                Whitespace
257            }
258            '/' => {
259                // Eat a line comment if it's `//...`
260                if self.eatc('/') {
261                    for (_, ch) in &mut self.chars {
262                        if ch == '\n' {
263                            break;
264                        }
265                    }
266                    Comment
267                // eat a block comment if it's `/*...`
268                } else if self.eatc('*') {
269                    let mut depth = 1;
270                    while depth > 0 {
271                        let (_, ch) = match self.chars.next() {
272                            Some(pair) => pair,
273                            None => return Err(Error::UnterminatedComment(start)),
274                        };
275                        match ch {
276                            '/' if self.eatc('*') => depth += 1,
277                            '*' if self.eatc('/') => depth -= 1,
278                            _ => {}
279                        }
280                    }
281                    Comment
282                } else {
283                    Slash
284                }
285            }
286            '=' => Equals,
287            ',' => Comma,
288            ':' => Colon,
289            '.' => Period,
290            ';' => Semicolon,
291            '(' => LeftParen,
292            ')' => RightParen,
293            '{' => LeftBrace,
294            '}' => RightBrace,
295            '<' => LessThan,
296            '>' => GreaterThan,
297            '*' => Star,
298            '@' => At,
299            '-' => {
300                if self.eatc('>') {
301                    RArrow
302                } else {
303                    Minus
304                }
305            }
306            '+' => Plus,
307            '%' => {
308                let mut iter = self.chars.clone();
309                if let Some((_, ch)) = iter.next() {
310                    if is_keylike_start(ch) {
311                        self.chars = iter.clone();
312                        while let Some((_, ch)) = iter.next() {
313                            if !is_keylike_continue(ch) {
314                                break;
315                            }
316                            self.chars = iter.clone();
317                        }
318                    }
319                }
320                ExplicitId
321            }
322            '"' => {
323                self.expect_string_literal(start)?;
324                StringLiteral
325            }
326            ch if is_keylike_start(ch) => {
327                let remaining = self.chars.chars.as_str().len();
328                let mut iter = self.chars.clone();
329                while let Some((_, ch)) = iter.next() {
330                    if !is_keylike_continue(ch) {
331                        break;
332                    }
333                    self.chars = iter.clone();
334                }
335                let str_end =
336                    str_start + ch.len_utf8() + (remaining - self.chars.chars.as_str().len());
337                match &self.input[str_start..str_end] {
338                    "use" => Use,
339                    "type" => Type,
340                    "func" => Func,
341                    "u8" => U8,
342                    "u16" => U16,
343                    "u32" => U32,
344                    "u64" => U64,
345                    "s8" => S8,
346                    "s16" => S16,
347                    "s32" => S32,
348                    "s64" => S64,
349                    "f32" => F32,
350                    "f64" => F64,
351                    "char" => Char,
352                    "resource" => Resource,
353                    "own" => Own,
354                    "borrow" => Borrow,
355                    "record" => Record,
356                    "flags" => Flags,
357                    "variant" => Variant,
358                    "enum" => Enum,
359                    "bool" => Bool,
360                    "string" => String_,
361                    "option" => Option_,
362                    "result" => Result_,
363                    "future" => Future,
364                    "stream" => Stream,
365                    "error-context" => ErrorContext,
366                    "list" => List,
367                    "map" => Map,
368                    "_" => Underscore,
369                    "as" => As,
370                    "from" => From_,
371                    "static" => Static,
372                    "interface" => Interface,
373                    "tuple" => Tuple,
374                    "world" => World,
375                    "import" => Import,
376                    "export" => Export,
377                    "package" => Package,
378                    "constructor" => Constructor,
379                    "include" => Include,
380                    "with" => With,
381                    "async" => Async,
382                    _ => Id,
383                }
384            }
385
386            ch if ch.is_ascii_digit() => {
387                let mut iter = self.chars.clone();
388                while let Some((_, ch)) = iter.next() {
389                    if !ch.is_ascii_digit() {
390                        break;
391                    }
392                    self.chars = iter.clone();
393                }
394
395                Integer
396            }
397
398            ch => return Err(Error::Unexpected(start, ch)),
399        };
400        let end = match self.chars.clone().next() {
401            Some((i, _)) => i,
402            None => self.input.len(),
403        };
404
405        let end = self.span_offset + u32::try_from(end).unwrap();
406        Ok(Some((Span::new(start, end), token)))
407    }
408
409    pub fn eat(&mut self, expected: Token) -> Result<bool, Error> {
410        let mut other = self.clone();
411        match other.next()? {
412            Some((_span, found)) if expected == found => {
413                *self = other;
414                Ok(true)
415            }
416            Some(_) => Ok(false),
417            None => Ok(false),
418        }
419    }
420
421    pub fn expect(&mut self, expected: Token) -> Result<Span, Error> {
422        match self.next()? {
423            Some((span, found)) => {
424                if expected == found {
425                    Ok(span)
426                } else {
427                    Err(Error::Wanted {
428                        at: span.start(),
429                        expected: expected.describe(),
430                        found: found.describe(),
431                    })
432                }
433            }
434            None => Err(Error::Wanted {
435                at: self.span_offset + u32::try_from(self.input.len()).unwrap(),
436                expected: expected.describe(),
437                found: "eof",
438            }),
439        }
440    }
441
442    fn eatc(&mut self, ch: char) -> bool {
443        let mut iter = self.chars.clone();
444        match iter.next() {
445            Some((_, ch2)) if ch == ch2 => {
446                self.chars = iter;
447                true
448            }
449            _ => false,
450        }
451    }
452
453    fn expect_any_char(&mut self) -> Result<(u32, char), Error> {
454        let end = self.eof_span().end;
455        let (pos, c) = self.chars.next().ok_or(Error::UnexpectedEof(end))?;
456        let pos = u32::try_from(pos).unwrap();
457        Ok((pos, c))
458    }
459
460    fn expect_char(&mut self, expected: char) -> Result<(), Error> {
461        let (pos, actual) = self.expect_any_char()?;
462        if actual == expected {
463            Ok(())
464        } else {
465            Err(Error::WantedChar(pos, expected))
466        }
467    }
468
469    pub fn string_literal(&mut self, span: Span) -> Result<String, Error> {
470        let input = self.get_span(span);
471        let input = &input[1..];
472        Tokenizer::new(input, span.start + 1)?.expect_string_literal(span.start + 1)
473    }
474
475    fn expect_string_literal(&mut self, start: u32) -> Result<String, Error> {
476        let mut buf = Vec::new();
477        loop {
478            let (pos, c) = self.expect_any_char()?;
479            match c {
480                '"' => break,
481                '\\' => {
482                    let (pos, c) = self.expect_any_char()?;
483                    match c {
484                        '"' => buf.push(b'"'),
485                        '\'' => buf.push(b'\''),
486                        't' => buf.push(b'\t'),
487                        'n' => buf.push(b'\n'),
488                        'r' => buf.push(b'\r'),
489                        '\\' => buf.push(b'\\'),
490                        'u' => {
491                            self.expect_char('{')?;
492                            let n = self.eat_hexnum()?;
493                            let c = char::from_u32(n).ok_or(Error::InvalidUnicodeValue(pos, n))?;
494                            buf.extend(c.encode_utf8(&mut [0; 4]).as_bytes());
495                            self.expect_char('}')?;
496                        }
497                        c1 if c1.is_ascii_hexdigit() => {
498                            let (_, c2) = self.eat_hexdigit()?;
499                            buf.push(to_hex(c1) * 16 + c2);
500                        }
501                        c => return Err(Error::InvalidStringEscape(pos, c)),
502                    }
503                }
504                c if (c as u32) < 0x20 || c as u32 == 0x7f => {
505                    return Err(Error::InvalidStringElement(pos, c));
506                }
507                c => buf.extend(c.encode_utf8(&mut [0; 4]).as_bytes()),
508            }
509        }
510        match String::from_utf8(buf) {
511            Ok(s) => Ok(s),
512            Err(e) => Err(Error::InvalidUtf8(start, e.utf8_error())),
513        }
514    }
515
516    pub fn eof_span(&self) -> Span {
517        let end = self.span_offset + u32::try_from(self.input.len()).unwrap();
518        Span::new(end, end)
519    }
520
521    fn eat_hexnum(&mut self) -> Result<u32, Error> {
522        let (pos, n) = self.eat_hexdigit()?;
523        let mut last_underscore = false;
524        let mut n = n as u32;
525        loop {
526            if self.eatc('_') {
527                last_underscore = true;
528                continue;
529            }
530            let (pos, c) = self.clone().expect_any_char()?;
531            if !c.is_ascii_hexdigit() {
532                break;
533            }
534            last_underscore = false;
535            self.chars.next();
536            n = n
537                .checked_mul(16)
538                .and_then(|n| n.checked_add(to_hex(c) as u32))
539                .ok_or(Error::NumberTooBig(pos))?;
540        }
541        if last_underscore {
542            return Err(Error::LoneUnderscore(pos));
543        }
544        Ok(n)
545    }
546
547    /// Reads a hexadecimal digit from the input stream, returning where it's
548    /// defined and the hex value. Returns an error on EOF or an invalid hex
549    /// digit.
550    fn eat_hexdigit(&mut self) -> Result<(u32, u8), Error> {
551        let (pos, ch) = self.expect_any_char()?;
552        if ch.is_ascii_hexdigit() {
553            Ok((pos, to_hex(ch)))
554        } else {
555            Err(Error::InvalidHexDigit(pos, ch))
556        }
557    }
558}
559
560fn to_hex(c: char) -> u8 {
561    match c {
562        'a'..='f' => c as u8 - b'a' + 10,
563        'A'..='F' => c as u8 - b'A' + 10,
564        _ => c as u8 - b'0',
565    }
566}
567
568impl<'a> Iterator for CrlfFold<'a> {
569    type Item = (usize, char);
570
571    fn next(&mut self) -> Option<(usize, char)> {
572        self.chars.next().map(|(i, c)| {
573            if c == '\r' {
574                let mut attempt = self.chars.clone();
575                if let Some((_, '\n')) = attempt.next() {
576                    self.chars = attempt;
577                    return (i, '\n');
578                }
579            }
580            (i, c)
581        })
582    }
583}
584
585fn detect_invalid_input(input: &str) -> Result<(), Error> {
586    // Disallow specific codepoints.
587    for (pos, ch) in input.char_indices() {
588        match ch {
589            '\n' | '\r' | '\t' => {}
590
591            // Bidirectional override codepoints can be used to craft source code that
592            // appears to have a different meaning than its actual meaning. See
593            // [CVE-2021-42574] for background and motivation.
594            //
595            // [CVE-2021-42574]: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-42574
596            '\u{202a}' | '\u{202b}' | '\u{202c}' | '\u{202d}' | '\u{202e}' | '\u{2066}'
597            | '\u{2067}' | '\u{2068}' | '\u{2069}' => {
598                return Err(Error::ForbiddenCodepoint(u32::try_from(pos).unwrap(), ch));
599            }
600
601            // Disallow several characters which are deprecated or discouraged in Unicode.
602            //
603            // U+149 deprecated; see Unicode 13.0.0, sec. 7.1 Latin, Compatibility Digraphs.
604            // U+673 deprecated; see Unicode 13.0.0, sec. 9.2 Arabic, Additional Vowel Marks.
605            // U+F77 and U+F79 deprecated; see Unicode 13.0.0, sec. 13.4 Tibetan, Vowels.
606            // U+17A3 and U+17A4 deprecated, and U+17B4 and U+17B5 discouraged; see
607            // Unicode 13.0.0, sec. 16.4 Khmer, Characters Whose Use Is Discouraged.
608            '\u{149}' | '\u{673}' | '\u{f77}' | '\u{f79}' | '\u{17a3}' | '\u{17a4}'
609            | '\u{17b4}' | '\u{17b5}' => {
610                return Err(Error::DeprecatedCodepoint(u32::try_from(pos).unwrap(), ch));
611            }
612
613            // Disallow control codes other than the ones explicitly recognized above,
614            // so that viewing a wit file on a terminal doesn't have surprising side
615            // effects or appear to have a different meaning than its actual meaning.
616            ch if ch.is_control() => {
617                return Err(Error::ControlCodepoint(u32::try_from(pos).unwrap(), ch));
618            }
619
620            _ => {}
621        }
622    }
623
624    Ok(())
625}
626
627fn is_keylike_start(ch: char) -> bool {
628    // Lex any XID start, `_`, or '-'. These aren't all valid identifier chars,
629    // but we'll diagnose that after we've lexed the full string.
630    unicode_ident::is_xid_start(ch) || ch == '_' || ch == '-'
631}
632
633fn is_keylike_continue(ch: char) -> bool {
634    // Lex any XID continue (which includes `_`) or '-'.
635    unicode_ident::is_xid_continue(ch) || ch == '-'
636}
637
638pub fn validate_id(start: u32, id: &str) -> Result<(), Error> {
639    // IDs must have at least one part.
640    if id.is_empty() {
641        return Err(Error::IdPartEmpty(start));
642    }
643
644    // Ids consist of parts separated by '-'s.
645    for (idx, part) in id.split('-').enumerate() {
646        // Parts must be non-empty and contain either all ASCII lowercase or
647        // all ASCII uppercase. Non-first segment can also start with a digit.
648        let Some(first_char) = part.chars().next() else {
649            return Err(Error::IdPartEmpty(start));
650        };
651        if idx == 0 && !first_char.is_ascii_alphabetic() {
652            return Err(Error::InvalidCharInId(start, first_char));
653        }
654        let mut upper = None;
655        for ch in part.chars() {
656            if ch.is_ascii_digit() {
657                // Digits are accepted in both uppercase and lowercase segments.
658            } else if ch.is_ascii_uppercase() {
659                if upper.is_none() {
660                    upper = Some(true);
661                } else if let Some(false) = upper {
662                    return Err(Error::InvalidCharInId(start, ch));
663                }
664            } else if ch.is_ascii_lowercase() {
665                if upper.is_none() {
666                    upper = Some(false);
667                } else if let Some(true) = upper {
668                    return Err(Error::InvalidCharInId(start, ch));
669                }
670            } else {
671                return Err(Error::InvalidCharInId(start, ch));
672            }
673        }
674    }
675
676    Ok(())
677}
678
679impl Token {
680    pub fn describe(&self) -> &'static str {
681        match self {
682            Whitespace => "whitespace",
683            Comment => "a comment",
684            Equals => "'='",
685            Comma => "','",
686            Colon => "':'",
687            Period => "'.'",
688            Semicolon => "';'",
689            LeftParen => "'('",
690            RightParen => "')'",
691            LeftBrace => "'{'",
692            RightBrace => "'}'",
693            LessThan => "'<'",
694            GreaterThan => "'>'",
695            Use => "keyword `use`",
696            Type => "keyword `type`",
697            Func => "keyword `func`",
698            U8 => "keyword `u8`",
699            U16 => "keyword `u16`",
700            U32 => "keyword `u32`",
701            U64 => "keyword `u64`",
702            S8 => "keyword `s8`",
703            S16 => "keyword `s16`",
704            S32 => "keyword `s32`",
705            S64 => "keyword `s64`",
706            F32 => "keyword `f32`",
707            F64 => "keyword `f64`",
708            Char => "keyword `char`",
709            Own => "keyword `own`",
710            Borrow => "keyword `borrow`",
711            Resource => "keyword `resource`",
712            Record => "keyword `record`",
713            Flags => "keyword `flags`",
714            Variant => "keyword `variant`",
715            Enum => "keyword `enum`",
716            Bool => "keyword `bool`",
717            String_ => "keyword `string`",
718            Option_ => "keyword `option`",
719            Result_ => "keyword `result`",
720            Future => "keyword `future`",
721            Stream => "keyword `stream`",
722            ErrorContext => "keyword `error-context`",
723            List => "keyword `list`",
724            Map => "keyword `map`",
725            Underscore => "keyword `_`",
726            Id => "an identifier",
727            ExplicitId => "an '%' identifier",
728            RArrow => "`->`",
729            Star => "`*`",
730            At => "`@`",
731            Slash => "`/`",
732            Plus => "`+`",
733            Minus => "`-`",
734            As => "keyword `as`",
735            From_ => "keyword `from`",
736            Static => "keyword `static`",
737            Interface => "keyword `interface`",
738            Tuple => "keyword `tuple`",
739            Import => "keyword `import`",
740            Export => "keyword `export`",
741            World => "keyword `world`",
742            Package => "keyword `package`",
743            Constructor => "keyword `constructor`",
744            Integer => "an integer",
745            Include => "keyword `include`",
746            With => "keyword `with`",
747            Async => "keyword `async`",
748            StringLiteral => "a string literal",
749        }
750    }
751}
752
753impl core::error::Error for Error {}
754
755impl Error {
756    /// Returns the byte offset in the source map where this error occurred.
757    pub fn position(&self) -> u32 {
758        match self {
759            Error::ControlCodepoint(at, _)
760            | Error::DeprecatedCodepoint(at, _)
761            | Error::ForbiddenCodepoint(at, _)
762            | Error::InvalidCharInId(at, _)
763            | Error::IdPartEmpty(at)
764            | Error::Unexpected(at, _)
765            | Error::UnterminatedComment(at)
766            | Error::InvalidUnicodeValue(at, _)
767            | Error::InvalidStringElement(at, _)
768            | Error::InvalidStringEscape(at, _)
769            | Error::WantedChar(at, _)
770            | Error::UnexpectedEof(at)
771            | Error::InvalidUtf8(at, _)
772            | Error::NumberTooBig(at)
773            | Error::LoneUnderscore(at)
774            | Error::InvalidHexDigit(at, _)
775            | Error::Wanted { at, .. } => *at,
776        }
777    }
778
779    pub(crate) fn adjust_position(&mut self, offset: u32) {
780        match self {
781            Error::ControlCodepoint(at, _)
782            | Error::DeprecatedCodepoint(at, _)
783            | Error::ForbiddenCodepoint(at, _)
784            | Error::InvalidCharInId(at, _)
785            | Error::IdPartEmpty(at)
786            | Error::Unexpected(at, _)
787            | Error::UnterminatedComment(at)
788            | Error::InvalidUnicodeValue(at, _)
789            | Error::InvalidStringElement(at, _)
790            | Error::InvalidStringEscape(at, _)
791            | Error::WantedChar(at, _)
792            | Error::UnexpectedEof(at)
793            | Error::InvalidUtf8(at, _)
794            | Error::NumberTooBig(at)
795            | Error::LoneUnderscore(at)
796            | Error::InvalidHexDigit(at, _)
797            | Error::Wanted { at, .. } => *at += offset,
798        }
799    }
800}
801
802impl fmt::Display for Error {
803    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
804        match self {
805            Error::ControlCodepoint(_, ch) => write!(f, "Control code '{}'", ch.escape_unicode()),
806            Error::DeprecatedCodepoint(_, ch) => {
807                write!(
808                    f,
809                    "Codepoint {:?} is discouraged by Unicode",
810                    ch.escape_unicode()
811                )
812            }
813            Error::ForbiddenCodepoint(_, ch) => {
814                write!(
815                    f,
816                    "Input contains bidirectional override codepoint {:?}",
817                    ch.escape_unicode()
818                )
819            }
820            Error::Unexpected(_, ch) => write!(f, "unexpected character {ch:?}"),
821            Error::UnterminatedComment(_) => write!(f, "unterminated block comment"),
822            Error::Wanted {
823                expected, found, ..
824            } => write!(f, "expected {expected}, found {found}"),
825            Error::InvalidCharInId(_, ch) => write!(f, "invalid character in identifier {ch:?}"),
826            Error::IdPartEmpty(_) => write!(f, "identifiers must have characters between '-'s"),
827            Error::InvalidUnicodeValue(_, val) => write!(f, "invalid unicode value {val:#x}"),
828            Error::InvalidStringElement(_, c) => write!(f, "invalid string character {c:?}"),
829            Error::InvalidStringEscape(_, c) => write!(f, "invalid string escape {c:?}"),
830            Error::WantedChar(_, c) => write!(f, "expected character {c:?}"),
831            Error::UnexpectedEof(_) => write!(f, "unexpected end of file"),
832            Error::InvalidUtf8(_, err) => write!(f, "invalid UTF-8: {err}"),
833            Error::NumberTooBig(_) => write!(f, "number is too big to fit in a u32"),
834            Error::LoneUnderscore(_) => write!(f, "trailing underscore in number"),
835            Error::InvalidHexDigit(_, c) => write!(f, "invalid hex digit {c:?}"),
836        }
837    }
838}
839
840#[test]
841fn test_validate_id() {
842    validate_id(0, "apple").unwrap();
843    validate_id(0, "apple-pear").unwrap();
844    validate_id(0, "apple-pear-grape").unwrap();
845    validate_id(0, "a0").unwrap();
846    validate_id(0, "a").unwrap();
847    validate_id(0, "a-a").unwrap();
848    validate_id(0, "bool").unwrap();
849    validate_id(0, "APPLE").unwrap();
850    validate_id(0, "APPLE-PEAR").unwrap();
851    validate_id(0, "APPLE-PEAR-GRAPE").unwrap();
852    validate_id(0, "apple-PEAR-grape").unwrap();
853    validate_id(0, "APPLE-pear-GRAPE").unwrap();
854    validate_id(0, "ENOENT").unwrap();
855    validate_id(0, "is-XML").unwrap();
856    validate_id(0, "apple-0").unwrap();
857    validate_id(0, "a0-000-3d4a-54FF").unwrap();
858
859    assert!(validate_id(0, "").is_err());
860    assert!(validate_id(0, "0").is_err());
861    assert!(validate_id(0, "%").is_err());
862    assert!(validate_id(0, "$").is_err());
863    assert!(validate_id(0, "0a").is_err());
864    assert!(validate_id(0, ".").is_err());
865    assert!(validate_id(0, "·").is_err());
866    assert!(validate_id(0, "a a").is_err());
867    assert!(validate_id(0, "_").is_err());
868    assert!(validate_id(0, "-").is_err());
869    assert!(validate_id(0, "a-").is_err());
870    assert!(validate_id(0, "-a").is_err());
871    assert!(validate_id(0, "Apple").is_err());
872    assert!(validate_id(0, "applE").is_err());
873    assert!(validate_id(0, "-apple-pear").is_err());
874    assert!(validate_id(0, "apple-pear-").is_err());
875    assert!(validate_id(0, "apple_pear").is_err());
876    assert!(validate_id(0, "apple.pear").is_err());
877    assert!(validate_id(0, "apple pear").is_err());
878    assert!(validate_id(0, "apple/pear").is_err());
879    assert!(validate_id(0, "apple|pear").is_err());
880    assert!(validate_id(0, "apple-Pear").is_err());
881    assert!(validate_id(0, "()()").is_err());
882    assert!(validate_id(0, "").is_err());
883    assert!(validate_id(0, "*").is_err());
884    assert!(validate_id(0, "apple\u{5f3}pear").is_err());
885    assert!(validate_id(0, "apple\u{200c}pear").is_err());
886    assert!(validate_id(0, "apple\u{200d}pear").is_err());
887    assert!(validate_id(0, "apple--pear").is_err());
888    assert!(validate_id(0, "_apple").is_err());
889    assert!(validate_id(0, "apple_").is_err());
890    assert!(validate_id(0, "_Znwj").is_err());
891    assert!(validate_id(0, "__i386").is_err());
892    assert!(validate_id(0, "__i386__").is_err());
893    assert!(validate_id(0, "Москва").is_err());
894    assert!(validate_id(0, "garçon-hühnervögel-Москва-東京").is_err());
895    assert!(validate_id(0, "a0-000-3d4A-54Ff").is_err());
896    assert!(validate_id(0, "😼").is_err(), "non-identifier");
897    assert!(validate_id(0, "\u{212b}").is_err(), "non-ascii");
898}
899
900#[test]
901fn test_tokenizer() {
902    fn collect(s: &str) -> Result<Vec<Token>, Error> {
903        let mut t = Tokenizer::new(s, 0)?;
904        let mut tokens = Vec::new();
905        while let Some(token) = t.next()? {
906            tokens.push(token.1);
907        }
908        Ok(tokens)
909    }
910
911    assert_eq!(collect("").unwrap(), vec![]);
912    assert_eq!(collect("_").unwrap(), vec![Token::Underscore]);
913    assert_eq!(collect("apple").unwrap(), vec![Token::Id]);
914    assert_eq!(collect("apple-pear").unwrap(), vec![Token::Id]);
915    assert_eq!(collect("apple--pear").unwrap(), vec![Token::Id]);
916    assert_eq!(collect("apple-Pear").unwrap(), vec![Token::Id]);
917    assert_eq!(collect("apple-pear-grape").unwrap(), vec![Token::Id]);
918    assert_eq!(collect("apple pear").unwrap(), vec![Token::Id, Token::Id]);
919    assert_eq!(collect("_a_p_p_l_e_").unwrap(), vec![Token::Id]);
920    assert_eq!(collect("garçon").unwrap(), vec![Token::Id]);
921    assert_eq!(collect("hühnervögel").unwrap(), vec![Token::Id]);
922    assert_eq!(collect("москва").unwrap(), vec![Token::Id]);
923    assert_eq!(collect("東京").unwrap(), vec![Token::Id]);
924    assert_eq!(
925        collect("garçon-hühnervögel-москва-東京").unwrap(),
926        vec![Token::Id]
927    );
928    assert_eq!(collect("a0").unwrap(), vec![Token::Id]);
929    assert_eq!(collect("a").unwrap(), vec![Token::Id]);
930    assert_eq!(collect("%a").unwrap(), vec![Token::ExplicitId]);
931    assert_eq!(collect("%a-a").unwrap(), vec![Token::ExplicitId]);
932    assert_eq!(collect("%bool").unwrap(), vec![Token::ExplicitId]);
933    assert_eq!(collect("%").unwrap(), vec![Token::ExplicitId]);
934    assert_eq!(collect("APPLE").unwrap(), vec![Token::Id]);
935    assert_eq!(collect("APPLE-PEAR").unwrap(), vec![Token::Id]);
936    assert_eq!(collect("APPLE-PEAR-GRAPE").unwrap(), vec![Token::Id]);
937    assert_eq!(collect("apple-PEAR-grape").unwrap(), vec![Token::Id]);
938    assert_eq!(collect("APPLE-pear-GRAPE").unwrap(), vec![Token::Id]);
939    assert_eq!(collect("ENOENT").unwrap(), vec![Token::Id]);
940    assert_eq!(collect("is-XML").unwrap(), vec![Token::Id]);
941
942    assert_eq!(collect("func").unwrap(), vec![Token::Func]);
943    assert_eq!(
944        collect("a: func()").unwrap(),
945        vec![
946            Token::Id,
947            Token::Colon,
948            Token::Func,
949            Token::LeftParen,
950            Token::RightParen
951        ]
952    );
953
954    assert_eq!(collect("resource").unwrap(), vec![Token::Resource]);
955
956    assert_eq!(collect("own").unwrap(), vec![Token::Own]);
957    assert_eq!(
958        collect("own<some-id>").unwrap(),
959        vec![Token::Own, Token::LessThan, Token::Id, Token::GreaterThan]
960    );
961
962    assert_eq!(collect("borrow").unwrap(), vec![Token::Borrow]);
963    assert_eq!(
964        collect("borrow<some-id>").unwrap(),
965        vec![
966            Token::Borrow,
967            Token::LessThan,
968            Token::Id,
969            Token::GreaterThan
970        ]
971    );
972
973    assert!(collect("\u{149}").is_err(), "strongly discouraged");
974    assert!(collect("\u{673}").is_err(), "strongly discouraged");
975    assert!(collect("\u{17a3}").is_err(), "strongly discouraged");
976    assert!(collect("\u{17a4}").is_err(), "strongly discouraged");
977    assert!(collect("\u{202a}").is_err(), "bidirectional override");
978    assert!(collect("\u{2068}").is_err(), "bidirectional override");
979    assert!(collect("\u{0}").is_err(), "control code");
980    assert!(collect("\u{b}").is_err(), "control code");
981    assert!(collect("\u{c}").is_err(), "control code");
982    assert!(collect("\u{85}").is_err(), "control code");
983}
984
985#[test]
986fn test_strings() {
987    #[track_caller]
988    fn test(s: &str, expected: Result<&str, Error>) {
989        let actual = (|| {
990            let mut t = Tokenizer::new(s, 0)?;
991            let next = t.next()?;
992            assert!(
993                matches!(next, Some((_, Token::StringLiteral))),
994                "{s:?} didn't tokenize as a string"
995            );
996            assert!(t.next()?.is_none(), "extra tokens after string: {s:?}");
997            let (span, _) = next.unwrap();
998            t.string_literal(span)
999        })();
1000        match (&actual, &expected) {
1001            (Ok(actual), Ok(expected)) => assert_eq!(actual, expected),
1002            (Err(actual), Err(expected)) => assert_eq!(actual, expected),
1003            (Ok(_) | Err(_), _) => panic!("expected error {expected:?}, but got Ok({actual:?})"),
1004        }
1005    }
1006
1007    // smoke test
1008    test("\"\"", Ok(""));
1009    test("\"a\"", Ok("a"));
1010    test("\"a b c\"", Ok("a b c"));
1011
1012    // single-char-escapes
1013    test("\"\\\"\"", Ok("\""));
1014    test("\"\\t\"", Ok("\t"));
1015    test("\"\\n\"", Ok("\n"));
1016    test("\"\\r\"", Ok("\r"));
1017    test("\"\\\\\"", Ok("\\"));
1018    test("\"\\h\"", Err(Error::InvalidStringEscape(2, 'h')));
1019
1020    // double-hex-digit
1021    test("\"\\00\"", Ok("\0"));
1022    test("\"\\01\"", Ok("\x01"));
1023    test("\"\\0f\"", Ok("\x0f"));
1024    test("\"\\0_1\"", Err(Error::InvalidHexDigit(3, '_')));
1025    test("\"\\0g\"", Err(Error::InvalidHexDigit(3, 'g')));
1026    #[allow(invalid_from_utf8)]
1027    test(
1028        "\"\\ff\"",
1029        Err(Error::InvalidUtf8(
1030            0,
1031            core::str::from_utf8(&[0xff]).unwrap_err(),
1032        )),
1033    );
1034
1035    // unicode escape
1036    test("\"\\u{0}\"", Ok("\0"));
1037    test("\"\\u{1}\"", Ok("\x01"));
1038    test("\"\\u{1_2_3_f}\"", Ok("\u{123f}"));
1039    test("\"\\u0000\"", Err(Error::WantedChar(3, '{')));
1040    test("\"\\u{0h\"", Err(Error::WantedChar(5, '}')));
1041    test("\"\\u{1_}\"", Err(Error::LoneUnderscore(4)));
1042    test(
1043        "\"\\u{fffffffffffffffffffff}\"",
1044        Err(Error::NumberTooBig(12)),
1045    );
1046    test(
1047        "\"\\u{ffff_ffff}\"",
1048        Err(Error::InvalidUnicodeValue(2, u32::MAX)),
1049    );
1050
1051    test("\"\t\"", Err(Error::InvalidStringElement(1, '\t')));
1052}
wit_parser/ast/lex.rs

wit_parser/ast/
lex.rs
