wolf_lexer/
lib.rs

1//! The wolfpack lexer: source text to tokens (spec chapter 1).
2//!
3//! The lexer is written by hand. It never stops at an error: every problem becomes a
4//! diagnostic, the offending text becomes an [`TokenKind::Error`] token, and lexing continues.
5//! It never panics on any input.
6
7mod cursor;
8mod lexer;
9mod token;
10
11use wolf_diag::{Diagnostic, codes};
12use wolf_span::{FileId, Span};
13
14pub use token::{Comment, FPart, Token, TokenKind};
15
16/// The result of lexing one file.
17#[derive(Debug, Default)]
18pub struct Lexed {
19    /// The tokens, always ending with [`TokenKind::Eof`].
20    pub tokens: Vec<Token>,
21    /// Ordinary comments, in source order.
22    pub comments: Vec<Comment>,
23    /// Lexical errors, in source order.
24    pub diagnostics: Vec<Diagnostic>,
25}
26
27/// Lexes the text of one file.
28pub fn lex(file: FileId, text: &str) -> Lexed {
29    lexer::Lexer::new(file, text).run()
30}
31
32/// A file whose bytes are not valid UTF-8 (spec chapter 1, rule 1.1).
33#[derive(Debug)]
34pub struct InvalidUtf8 {
35    /// The text with invalid sequences replaced by U+FFFD, for showing the error.
36    pub lossy: String,
37    /// Byte offset of the first invalid sequence, in both the original bytes and `lossy`.
38    pub offset: usize,
39}
40
41impl InvalidUtf8 {
42    /// The E0101 diagnostic for this file, once `lossy` has been added to the source map as
43    /// `file`.
44    pub fn diagnostic(&self, file: FileId) -> Diagnostic {
45        let start = u32::try_from(self.offset).unwrap_or(u32::MAX);
46        // U+FFFD takes 3 bytes in `lossy`, so the label covers a whole character.
47        let end = start.saturating_add(3);
48        Diagnostic::new(codes::E0101, "file is not valid UTF-8")
49            .primary(Span::new(file, start, end), "invalid byte sequence here")
50            .note("Wolfpack source files must be encoded in UTF-8")
51    }
52}
53
54/// Decodes the bytes of a source file. A byte order mark is kept; the lexer skips it.
55pub fn decode(bytes: Vec<u8>) -> Result<String, InvalidUtf8> {
56    match String::from_utf8(bytes) {
57        Ok(text) => Ok(text),
58        Err(error) => {
59            let offset = error.utf8_error().valid_up_to();
60            let lossy = String::from_utf8_lossy(error.as_bytes()).into_owned();
61            Err(InvalidUtf8 { lossy, offset })
62        }
63    }
64}
65
66/// Renders tokens one per line, for snapshot tests and `wolf check --emit tokens`.
67pub fn dump(text: &str, tokens: &[Token]) -> String {
68    let mut out = String::new();
69    for token in tokens {
70        let source = text.get(token.span.range()).unwrap_or("");
71        let kind = match &token.kind {
72            TokenKind::Ident => format!("Ident {source:?}"),
73            TokenKind::Int(value) => format!("Int {value}"),
74            TokenKind::Float(value) => format!("Float {value:?}"),
75            TokenKind::Str(value) => format!("Str {value:?}"),
76            TokenKind::FStr(parts) => {
77                let parts: Vec<String> = parts
78                    .iter()
79                    .map(|part| match part {
80                        FPart::Text(text) => format!("{text:?}"),
81                        FPart::Expr(tokens, _) => format!("{{{}}}", tokens.len().saturating_sub(1)),
82                    })
83                    .collect();
84                format!("FStr [{}]", parts.join(", "))
85            }
86            TokenKind::DocOuter(value) => format!("DocOuter {value:?}"),
87            TokenKind::DocInner(value) => format!("DocInner {value:?}"),
88            TokenKind::Error => format!("Error {source:?}"),
89            other => match other.spelling() {
90                Some(spelling) => spelling.to_string(),
91                None => format!("{other:?}"),
92            },
93        };
94        out.push_str(&format!(
95            "{}..{} {}\n",
96            token.span.start, token.span.end, kind
97        ));
98    }
99    out
100}
wolf_lexer/lib.rs

wolf_lexer/
lib.rs
