badness_parser/bib/
lexer.rs

1//! A total, lossless lexer for BibTeX/BibLaTeX surface syntax.
2//!
3//! Every byte of the input ends up in exactly one token, so concatenating all
4//! token texts reproduces the input verbatim — the losslessness invariant.
5//!
6//! The lexer is context-free: brace/quote *structure* is the parser's job (it
7//! tracks brace depth to build `BRACE_GROUP` / `QUOTED` value nodes), exactly as
8//! the LaTeX lexer leaves `{`/`}` grouping to its grammar. BibTeX needs no
9//! verbatim or catcode modes, so this lexer is simpler than the LaTeX one.
10//!
11//! The specials `@ { } ( ) , = # "` each lex as a single-character token; runs of
12//! whitespace, line breaks, and "word" characters (anything else) coalesce. A
13//! word run made up solely of ASCII digits is classified [`SyntaxKind::NUMBER`],
14//! else [`SyntaxKind::WORD`] — so a later formatter / linter can tell an unquoted
15//! number from a macro name.
16
17use smol_str::SmolStr;
18
19use crate::bib::syntax::SyntaxKind;
20
21/// A single lexed token: its kind plus the exact source slice it covers.
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub struct Token {
24    pub kind: SyntaxKind,
25    pub text: SmolStr,
26}
27
28/// Is `c` one of the single-character special tokens?
29fn special_kind(c: u8) -> Option<SyntaxKind> {
30    Some(match c {
31        b'@' => SyntaxKind::AT,
32        b'{' => SyntaxKind::L_BRACE,
33        b'}' => SyntaxKind::R_BRACE,
34        b'(' => SyntaxKind::L_PAREN,
35        b')' => SyntaxKind::R_PAREN,
36        b',' => SyntaxKind::COMMA,
37        b'=' => SyntaxKind::EQ,
38        b'#' => SyntaxKind::HASH,
39        b'"' => SyntaxKind::QUOTE,
40        _ => return None,
41    })
42}
43
44/// Does `c` end a word run? (whitespace, a line break, or a special)
45fn is_word_boundary(c: u8) -> bool {
46    matches!(c, b' ' | b'\t' | b'\n' | b'\r') || special_kind(c).is_some()
47}
48
49/// Lex `input` into a flat, lossless token stream.
50pub fn lex(input: &str) -> Vec<Token> {
51    let bytes = input.as_bytes();
52    let mut out = Vec::new();
53    let mut pos = 0;
54    while pos < bytes.len() {
55        let c = bytes[pos];
56        let (kind, len) = if let Some(kind) = special_kind(c) {
57            (kind, 1)
58        } else if c == b'\n' {
59            (SyntaxKind::NEWLINE, 1)
60        } else if c == b'\r' {
61            // `\r\n` is one line break; a lone `\r` is its own.
62            let len = if bytes.get(pos + 1) == Some(&b'\n') {
63                2
64            } else {
65                1
66            };
67            (SyntaxKind::NEWLINE, len)
68        } else if c == b' ' || c == b'\t' {
69            let len = run_len(bytes, pos, |b| b == b' ' || b == b'\t');
70            (SyntaxKind::WHITESPACE, len)
71        } else {
72            let len = run_len(bytes, pos, |b| !is_word_boundary(b));
73            let kind = if bytes[pos..pos + len].iter().all(u8::is_ascii_digit) {
74                SyntaxKind::NUMBER
75            } else {
76                SyntaxKind::WORD
77            };
78            (kind, len)
79        };
80        out.push(Token {
81            kind,
82            text: SmolStr::new(&input[pos..pos + len]),
83        });
84        pos += len;
85    }
86    out
87}
88
89/// Length of the maximal run of bytes from `start` satisfying `pred`. The byte at
90/// `start` is assumed to satisfy `pred`, so the run is always at least one byte.
91fn run_len(bytes: &[u8], start: usize, pred: impl Fn(u8) -> bool) -> usize {
92    let mut i = start + 1;
93    while i < bytes.len() && pred(bytes[i]) {
94        i += 1;
95    }
96    i - start
97}
badness_parser/bib/lexer.rs

badness_parser/bib/
lexer.rs
