badness_parser/bib/
lexer.rs

1//! A total, lossless lexer for BibTeX/BibLaTeX surface syntax.
2//!
3//! Every byte of the input ends up in exactly one token, so concatenating all
4//! token texts reproduces the input verbatim — the losslessness invariant.
5//!
6//! The lexer is context-free: brace/quote *structure* is the parser's job (it
7//! tracks brace depth to build `BRACE_GROUP` / `QUOTED` value nodes), exactly as
8//! the LaTeX lexer leaves `{`/`}` grouping to its grammar. BibTeX needs no
9//! verbatim or catcode modes, so this lexer is simpler than the LaTeX one.
10//!
11//! The specials `@ { } ( ) , = # " %` each lex as a single-character token; runs
12//! of whitespace, line breaks, and "word" characters (anything else) coalesce. A
13//! word run made up solely of ASCII digits is classified [`SyntaxKind::NUMBER`],
14//! else [`SyntaxKind::WORD`] — so a later formatter / linter can tell an unquoted
15//! number from a macro name.
16//!
17//! `%` is a *bare* [`SyntaxKind::PERCENT`] token here, not a comment: whether it
18//! opens a comment depends on brace/quote context (`{50% off}` is literal text),
19//! and that context is the grammar's to know. The grammar wraps the run from a
20//! `%` to the end of its line in a [`SyntaxKind::COMMENT`] node only where BibTeX
21//! allows one.
22
23use smol_str::SmolStr;
24
25use crate::bib::syntax::SyntaxKind;
26
27/// A single lexed token: its kind plus the exact source slice it covers.
28#[derive(Debug, Clone, PartialEq, Eq)]
29pub struct Token {
30    pub kind: SyntaxKind,
31    pub text: SmolStr,
32}
33
34/// Is `c` one of the single-character special tokens?
35fn special_kind(c: u8) -> Option<SyntaxKind> {
36    Some(match c {
37        b'@' => SyntaxKind::AT,
38        b'{' => SyntaxKind::L_BRACE,
39        b'}' => SyntaxKind::R_BRACE,
40        b'(' => SyntaxKind::L_PAREN,
41        b')' => SyntaxKind::R_PAREN,
42        b',' => SyntaxKind::COMMA,
43        b'=' => SyntaxKind::EQ,
44        b'#' => SyntaxKind::HASH,
45        b'"' => SyntaxKind::QUOTE,
46        b'%' => SyntaxKind::PERCENT,
47        _ => return None,
48    })
49}
50
51/// Does `c` end a word run? (whitespace, a line break, or a special)
52fn is_word_boundary(c: u8) -> bool {
53    matches!(c, b' ' | b'\t' | b'\n' | b'\r') || special_kind(c).is_some()
54}
55
56/// Lex `input` into a flat, lossless token stream.
57pub fn lex(input: &str) -> Vec<Token> {
58    let bytes = input.as_bytes();
59    let mut out = Vec::new();
60    let mut pos = 0;
61    while pos < bytes.len() {
62        let c = bytes[pos];
63        let (kind, len) = if let Some(kind) = special_kind(c) {
64            (kind, 1)
65        } else if c == b'\n' {
66            (SyntaxKind::NEWLINE, 1)
67        } else if c == b'\r' {
68            // `\r\n` is one line break; a lone `\r` is its own.
69            let len = if bytes.get(pos + 1) == Some(&b'\n') {
70                2
71            } else {
72                1
73            };
74            (SyntaxKind::NEWLINE, len)
75        } else if c == b' ' || c == b'\t' {
76            let len = run_len(bytes, pos, |b| b == b' ' || b == b'\t');
77            (SyntaxKind::WHITESPACE, len)
78        } else {
79            let len = run_len(bytes, pos, |b| !is_word_boundary(b));
80            let kind = if bytes[pos..pos + len].iter().all(u8::is_ascii_digit) {
81                SyntaxKind::NUMBER
82            } else {
83                SyntaxKind::WORD
84            };
85            (kind, len)
86        };
87        out.push(Token {
88            kind,
89            text: SmolStr::new(&input[pos..pos + len]),
90        });
91        pos += len;
92    }
93    out
94}
95
96/// Length of the maximal run of bytes from `start` satisfying `pred`. The byte at
97/// `start` is assumed to satisfy `pred`, so the run is always at least one byte.
98fn run_len(bytes: &[u8], start: usize, pred: impl Fn(u8) -> bool) -> usize {
99    let mut i = start + 1;
100    while i < bytes.len() && pred(bytes[i]) {
101        i += 1;
102    }
103    i - start
104}
badness_parser/bib/lexer.rs

badness_parser/bib/
lexer.rs
