weavatrix_parse/token/
mod.rs

1//! Lossless tokenizer.
2//!
3//! Every byte of the input belongs to exactly one token, including whitespace
4//! and comments. Concatenating the text of all tokens reproduces the source
5//! exactly - a property the tests assert - so the same token stream serves a
6//! compiler front end, a formatter and an evidence extractor. Skipping trivia
7//! would make the stream cheaper and permanently unable to round-trip.
8
9use crate::syntax::{Language, Syntax};
10
11/// What a token is, at the lexical level only.
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13#[non_exhaustive]
14pub enum TokenKind {
15    /// Spaces and tabs; newlines are their own kind.
16    Whitespace,
17    Newline,
18    /// Leading whitespace of a line in an indentation-sensitive language.
19    Indent,
20    LineComment,
21    BlockComment,
22    /// String, character, template or raw-string literal, quotes included.
23    String,
24    /// Interpolated section of a template literal, `${` and `}` included.
25    Interpolation,
26    Number,
27    Identifier,
28    /// Regular-expression literal in languages that have them.
29    Regex,
30    Punctuation,
31    /// A block comment or string that the file ends inside.
32    Unterminated,
33}
34
35/// One lexical unit and its exact position in the source.
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub struct Token {
38    pub kind: TokenKind,
39    /// Byte range in the source; `source[start..end]` is the token text.
40    pub start: usize,
41    pub end: usize,
42    /// One-based line of the first byte.
43    pub line: u32,
44    /// One-based column, counted in characters, of the first byte.
45    pub column: u32,
46}
47
48impl Token {
49    /// The token's text.
50    #[must_use]
51    pub fn text<'source>(&self, source: &'source str) -> &'source str {
52        source.get(self.start..self.end).unwrap_or_default()
53    }
54
55    /// Whether this token carries no program meaning.
56    #[must_use]
57    pub const fn is_trivia(&self) -> bool {
58        matches!(
59            self.kind,
60            TokenKind::Whitespace
61                | TokenKind::Newline
62                | TokenKind::Indent
63                | TokenKind::LineComment
64                | TokenKind::BlockComment
65        )
66    }
67}
68
69/// How much of the source the stream carries.
70///
71/// The two modes exist because the consumers genuinely differ. A compiler
72/// front end, a formatter or a source-to-source translator must be able to
73/// rebuild the input, so they need every byte. Evidence extraction throws
74/// trivia away immediately, so carrying it only costs allocations.
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
76pub enum Mode {
77    /// Every byte belongs to a token; the stream rebuilds the source exactly.
78    #[default]
79    Lossless,
80    /// Whitespace and comments are skipped. Positions stay exact, so spans
81    /// remain usable, but the stream no longer round-trips.
82    Lite,
83}
84
85/// Tokenizes a whole source file losslessly.
86#[must_use]
87pub fn tokenize(source: &str, language: Language) -> Vec<Token> {
88    Tokenizer::new(source, language).collect()
89}
90
91/// Tokenizes a source file, dropping trivia.
92#[must_use]
93pub fn tokenize_lite(source: &str, language: Language) -> Vec<Token> {
94    Tokenizer::new(source, language).mode(Mode::Lite).collect()
95}
96
97/// Streaming tokenizer over one source file.
98pub struct Tokenizer<'source> {
99    source: &'source str,
100    syntax: Syntax,
101    mode: Mode,
102    bytes: &'source [u8],
103    offset: usize,
104    line: u32,
105    column: u32,
106    /// Whether the previous meaningful token can end an expression, which is
107    /// what decides between division and a regular-expression literal.
108    value_before: bool,
109    at_line_start: bool,
110}
111
112mod construction;
113mod lexemes;
114mod scanner;
115mod strings;
116
117#[cfg(test)]
118mod tests;
weavatrix_parse/token/mod.rs

weavatrix_parse/token/
mod.rs
