sql_dialect_fmt_lexer/
token.rs

1//! The token types produced by the lexer.
2
3use sql_dialect_fmt_syntax::SyntaxKind;
4
5/// A single lexed token. `text` borrows the source, so the lexer allocates nothing per token.
6#[derive(Clone, Copy, PartialEq, Eq, Debug)]
7pub struct Token<'a> {
8    pub kind: SyntaxKind,
9    pub text: &'a str,
10}
11
12/// A lexical diagnostic (e.g. an unterminated literal), located at a byte span.
13#[derive(Clone, PartialEq, Eq, Debug)]
14pub struct LexError {
15    pub message: String,
16    /// Byte offset into the source where the offending token began.
17    pub offset: usize,
18    /// Byte length of the offending token's text, so a diagnostic can underline the whole token
19    /// rather than a single character. May be `0` for a zero-width point at end of input.
20    pub len: usize,
21}
22
23impl LexError {
24    /// The byte range `offset..offset + len` this error covers in the source.
25    pub fn range(&self) -> std::ops::Range<usize> {
26        self.offset..self.offset + self.len
27    }
28}
29
30impl std::fmt::Display for LexError {
31    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
32        write!(f, "{} at byte {}", self.message, self.offset)
33    }
34}
35
36impl std::error::Error for LexError {}
37
38/// The result of lexing: the token stream plus any diagnostics.
39#[derive(Clone, Debug, Default)]
40pub struct Lexed<'a> {
41    pub tokens: Vec<Token<'a>>,
42    pub errors: Vec<LexError>,
43}
sql_dialect_fmt_lexer/token.rs

sql_dialect_fmt_lexer/
token.rs
