sql_dialect_fmt_lexer/
token.rs

1//! The token types produced by the lexer.
2
3use sql_dialect_fmt_syntax::SyntaxKind;
4use sql_dialect_fmt_text::LineColumn;
5
6/// A single lexed token. `text` borrows the source, so the lexer allocates nothing per token.
7#[derive(Clone, Copy, PartialEq, Eq, Debug)]
8pub struct Token<'a> {
9    pub kind: SyntaxKind,
10    pub text: &'a str,
11}
12
13/// A lexical diagnostic (e.g. an unterminated literal), located at a byte span.
14#[derive(Clone, PartialEq, Eq, Debug)]
15pub struct LexError {
16    pub message: String,
17    /// Byte offset into the source where the offending token began.
18    pub offset: usize,
19    /// Byte length of the offending token's text, so a diagnostic can underline the whole token
20    /// rather than a single character. May be `0` for a zero-width point at end of input.
21    pub len: usize,
22    /// One-based line/column position, when the full source text was available.
23    pub line_column: Option<LineColumn>,
24}
25
26impl LexError {
27    /// The byte range `offset..offset + len` this error covers in the source.
28    pub fn range(&self) -> std::ops::Range<usize> {
29        self.offset..self.offset + self.len
30    }
31}
32
33impl std::fmt::Display for LexError {
34    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
35        match self.line_column {
36            Some(pos) => write!(
37                f,
38                "{} at line {}, column {} (byte {})",
39                self.message, pos.line, pos.column, self.offset
40            ),
41            None => write!(f, "{} at byte {}", self.message, self.offset),
42        }
43    }
44}
45
46impl std::error::Error for LexError {}
47
48/// The result of lexing: the token stream plus any diagnostics.
49#[derive(Clone, Debug, Default)]
50pub struct Lexed<'a> {
51    pub tokens: Vec<Token<'a>>,
52    pub errors: Vec<LexError>,
53}
sql_dialect_fmt_lexer/token.rs

sql_dialect_fmt_lexer/
token.rs
