sql_dialect_fmt_lexer/
lib.rs

1//! Hand-written, lossless, error-resilient lexer for Snowflake SQL.
2//!
3//! Design goals (mirroring Biome / rust-analyzer):
4//! * **Lossless** — every byte of the input belongs to exactly one token, so the original
5//!   source can always be reconstructed by concatenating token texts. This is what makes
6//!   accurate formatting, comment preservation, and syntax highlighting possible.
7//! * **Error-resilient** — malformed input (unterminated strings/comments) still produces
8//!   tokens plus diagnostics, never a panic. Mid-edit SQL must still lex.
9//! * **Fast** — single pass over the bytes, zero allocation per token (texts borrow the
10//!   input), no regex.
11//!
12//! The lexer is deliberately "dumb": it emits [`SyntaxKind::IDENT`] for every keyword-like
13//! word. The parser reclassifies keywords contextually via [`sql_dialect_fmt_syntax::keyword_kind`],
14//! because many SQL keywords (LEFT, ROW, VALUE, ...) are contextual and may be identifiers.
15//!
16//! ## Modules
17//! * `token` — the [`Token`] / [`LexError`] / [`Lexed`] types.
18//! * `lexer` — the single-pass tokenizer ([`tokenize`]).
19
20mod delimiter;
21mod lexer;
22mod token;
23
24pub use delimiter::{BodyDelimiter, LexOptions, DEFAULT_BODY_DELIMITERS, DOLLAR_QUOTED_BODY};
25pub use lexer::{tokenize, tokenize_for_dialect, tokenize_with_options};
26pub use token::{LexError, Lexed, Token};
27
28// Re-exported so downstream crates and integration tests can name the kind through the lexer.
29pub use sql_dialect_fmt_syntax::{Dialect, SyntaxKind};
sql_dialect_fmt_lexer/lib.rs

sql_dialect_fmt_lexer/
lib.rs
