Skip to main content

buffa/text/
token.rs

1//! Textproto tokenizer.
2//!
3//! Produces a flat stream of [`Token`]s borrowing from the input `&str`. The
4//! tokenizer is **syntactic only**: it recognises the shape of numbers, string
5//! literals, and identifiers but does not interpret them. `42`, `"foo"`, and
6//! `true` all come out as [`TokenKind::Scalar`] with `raw` pointing at the
7//! relevant slice. Type-directed interpretation (is this i32? enum variant?
8//! bool?) happens in the decoder layer, so `expected i32, got string` errors
9//! are possible.
10//!
11//! The grammar is a simplified state machine driven by the last-returned token
12//! kind — reference implementation is protobuf-go's
13//! `internal/encoding/text/decode.go`. Colons, commas, and semicolons between
14//! tokens are consumed internally and never surfaced.
15
16use super::error::{ParseError, ParseErrorKind};
17
18/// The kind of a textproto token.
19///
20/// Colons, commas, and semicolons are consumed by the tokenizer between
21/// tokens and are not exposed.
22#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub enum TokenKind {
24    /// End of input.
25    Eof,
26    /// A field name: either a bare identifier (`foo`), a bracketed extension
27    /// name (`[pkg.ext]`), a bracketed Any URL (`[type.googleapis.com/Foo]`),
28    /// or a decimal field number (`42`).
29    Name,
30    /// A scalar value: number literal, string literal, enum identifier, or
31    /// `true`/`false`/`inf`/`nan`. Uninterpreted — the decoder parses `raw`
32    /// according to the expected field type.
33    Scalar,
34    /// `{` or `<` — start of a nested message.
35    MessageOpen,
36    /// `}` or `>` — end of a nested message.
37    MessageClose,
38    /// `[` — start of a list of scalar or message values.
39    ListOpen,
40    /// `]` — end of a list.
41    ListClose,
42}
43
44/// Secondary classification of a [`TokenKind::Name`] token.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum NameKind {
47    /// Bare identifier: `foo_bar`.
48    Ident,
49    /// Bracketed type name: `[pkg.ext]` or `[type.googleapis.com/pkg.Msg]`.
50    /// The brackets are included in `raw`.
51    TypeName,
52    /// Decimal integer used as a field number: `42`.
53    FieldNumber,
54}
55
56/// Secondary classification of a [`TokenKind::Scalar`] token.
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum ScalarKind {
59    /// A numeric literal: integer (dec/hex/oct) or float.
60    Number,
61    /// One or more adjacent quoted string literals.
62    String,
63    /// An identifier in value position: enum variant name, `true`, `false`,
64    /// `inf`, `nan`, etc. May have a leading `-`.
65    Literal,
66}
67
68/// A single textproto token, borrowing from the input.
69#[derive(Debug, Clone, Copy)]
70pub struct Token<'a> {
71    /// What kind of token this is.
72    pub kind: TokenKind,
73    /// The raw input slice covered by this token. For strings, this includes
74    /// the quotes and any adjacent concatenated literals. For bracketed type
75    /// names, it includes the `[` and `]`.
76    pub raw: &'a str,
77    /// Byte offset of `raw` into the original input. Use
78    /// [`Tokenizer::line_col`] to convert to 1-based line:col for errors.
79    pub pos: usize,
80    /// For `Name` tokens: which kind of name. `Ident` for non-`Name` tokens.
81    pub name_kind: NameKind,
82    /// For `Scalar` tokens: which kind of scalar. `Number` for non-`Scalar`.
83    pub scalar_kind: ScalarKind,
84    /// For `Name` tokens: was this name followed by a `:` separator?
85    /// The `:` is optional before a message value or a list of message
86    /// values, and required before a scalar or a list of scalars.
87    pub has_separator: bool,
88}
89
90/// Internal state: what the tokenizer is currently nested inside.
91#[derive(Clone, Copy, PartialEq, Eq)]
92enum OpenKind {
93    /// Not inside anything — top-level message body.
94    Top,
95    /// Inside `{ ... }` (close with `}`) or `< ... >` (close with `>`).
96    /// Holds the expected close byte.
97    Message(u8),
98    /// Inside `[ ... ]`. `scalars` is false for a list opened without a `:`
99    /// after the field name, which may hold only message values.
100    List { scalars: bool },
101}
102
103/// `open_stack` byte for a list opened without a `:` after the field name.
104/// The input byte is `[` for both kinds of list; this one never appears in
105/// the input as an open delimiter.
106const MESSAGE_LIST: u8 = b'(';
107
108/// `expected` text of the error for a scalar that follows a field name, or
109/// opens a list element, without a `:` after the name.
110/// `TextDecoder::read_repeated_scalar_into` reports the same text at the `[`
111/// of an empty list that follows such a name.
112pub(super) const COLON_BEFORE_SCALAR: &str = "':' before scalar value";
113
114/// Internal: the last-emitted token kind, driving the state machine.
115/// Slightly richer than [`TokenKind`] because it tracks the consumed
116/// separators.
117#[derive(Clone, Copy, PartialEq, Eq)]
118enum LastKind {
119    /// Beginning of file — no token emitted yet.
120    Bof,
121    Name(bool),
122    Scalar,
123    MessageOpen,
124    MessageClose,
125    ListOpen,
126    ListClose,
127    Comma,
128    Semicolon,
129}
130
131/// Stateful textproto tokenizer.
132///
133/// Holds a cursor into the input string and a small stack of open delimiters.
134/// [`peek`](Self::peek) caches its result so it is cheap to call repeatedly
135/// before [`read`](Self::read).
136pub struct Tokenizer<'a> {
137    input: &'a str,
138    cursor: usize,
139    last_kind: LastKind,
140    /// Stack of `{`, `<`, `[` and [`MESSAGE_LIST`] bytes. Capped at
141    /// [`RECURSION_LIMIT`](crate::RECURSION_LIMIT) — pushing beyond that
142    /// fails with [`ParseErrorKind::RecursionLimitExceeded`].
143    open_stack: [u8; crate::message::RECURSION_LIMIT as usize],
144    open_depth: usize,
145    /// Peek cache: last-produced token + its `last_kind` + cursor after it.
146    peeked: Option<(Token<'a>, LastKind, usize)>,
147}
148
149impl<'a> Tokenizer<'a> {
150    /// Create a tokenizer positioned at the start of `input`.
151    ///
152    /// A leading UTF-8 byte-order mark (`U+FEFF`) is silently skipped — some
153    /// editors prepend one when saving, and it is not a valid identifier
154    /// start character.
155    pub fn new(input: &'a str) -> Self {
156        // BOM is 3 bytes in UTF-8 (EF BB BF).
157        let cursor = if input.starts_with('\u{FEFF}') { 3 } else { 0 };
158        Tokenizer {
159            input,
160            cursor,
161            last_kind: LastKind::Bof,
162            open_stack: [0u8; crate::message::RECURSION_LIMIT as usize],
163            open_depth: 0,
164            peeked: None,
165        }
166    }
167
168    /// Look at the next token without consuming it.
169    ///
170    /// The result is cached: repeated `peek()` calls do not re-parse.
171    ///
172    /// # Errors
173    ///
174    /// Any tokenization error at the upcoming position.
175    pub fn peek(&mut self) -> Result<Token<'a>, ParseError> {
176        if let Some((tok, _, _)) = self.peeked {
177            return Ok(tok);
178        }
179        // Speculatively tokenize: save scalar state, run `parse_next`, restore.
180        //
181        // open_stack invariant: `parse_next` may *write* to
182        // `open_stack[save_depth]` (when it sees an open delimiter) or *read*
183        // `open_stack[save_depth - 1]` (when it sees a close). It never
184        // touches any other slot. Restoring `open_depth` here leaves the
185        // written byte in place — harmless, because the slot is beyond
186        // depth and unread by `current_open()`. When `read()` commits the
187        // cached token below, it re-increments depth and the byte is
188        // exactly where it needs to be. The single-token cache means we
189        // can never have two speculative pushes at different depths
190        // without an intervening commit.
191        let save_cursor = self.cursor;
192        let save_depth = self.open_depth;
193        let save_last = self.last_kind;
194        let tok = self.parse_next()?;
195        let new_last = self.last_kind;
196        let new_cursor = self.cursor;
197        // Restore — `read()` commits from the cache.
198        self.cursor = save_cursor;
199        self.open_depth = save_depth;
200        self.last_kind = save_last;
201        self.peeked = Some((tok, new_last, new_cursor));
202        Ok(tok)
203    }
204
205    /// Consume and return the next token.
206    ///
207    /// # Errors
208    ///
209    /// Any tokenization error at the current position.
210    pub fn read(&mut self) -> Result<Token<'a>, ParseError> {
211        if let Some((tok, last, cur)) = self.peeked.take() {
212            // Commit the peeked state. `peek()` rolled back `open_depth`
213            // but left the speculatively-written stack byte in place; we
214            // only need to re-apply the depth delta. See the invariant
215            // comment in `peek()`.
216            self.last_kind = last;
217            self.cursor = cur;
218            match tok.kind {
219                TokenKind::MessageOpen | TokenKind::ListOpen => self.open_depth += 1,
220                TokenKind::MessageClose | TokenKind::ListClose => self.open_depth -= 1,
221                _ => {}
222            }
223            return Ok(tok);
224        }
225        self.parse_next()
226    }
227
228    /// Convert a byte offset into the input into a 1-based (line, column).
229    ///
230    /// Column counts Unicode scalar values, not bytes. A `pos` past the end
231    /// of the input gives the position just after the last character. An
232    /// offset inside a UTF-8 character is treated as that character's start.
233    ///
234    /// Line and column use `u32`: inputs larger than ~4 GiB would wrap, but
235    /// that is not a realistic textproto size and this is error-reporting only.
236    pub fn line_col(&self, pos: usize) -> (u32, u32) {
237        let mut pos = pos.min(self.input.len());
238        while !self.input.is_char_boundary(pos) {
239            pos -= 1;
240        }
241        let before = &self.input[..pos];
242        let line = before.bytes().filter(|&b| b == b'\n').count() as u32 + 1;
243        let line_start = before.rfind('\n').map(|i| i + 1).unwrap_or(0);
244        let col = before[line_start..].chars().count() as u32 + 1;
245        (line, col)
246    }
247
248    // ── internals ───────────────────────────────────────────────────────────
249
250    fn err(&self, pos: usize, kind: ParseErrorKind) -> ParseError {
251        let (line, col) = self.line_col(pos);
252        ParseError::new(line, col, kind)
253    }
254
255    /// Whether the innermost open delimiter is a list that follows a field
256    /// name without a `:`.
257    ///
258    /// Valid once the list's `[` is read; a peek at the token after it leaves
259    /// the answer unchanged.
260    pub(super) fn in_list_without_separator(&self) -> bool {
261        self.current_open() == (OpenKind::List { scalars: false })
262    }
263
264    pub(super) fn err_here(&self, kind: ParseErrorKind) -> ParseError {
265        self.err(self.cursor, kind)
266    }
267
268    #[inline]
269    fn rest(&self) -> &'a [u8] {
270        &self.input.as_bytes()[self.cursor..]
271    }
272
273    /// What we're currently nested inside, based on the top of the open stack.
274    fn current_open(&self) -> OpenKind {
275        if self.open_depth == 0 {
276            return OpenKind::Top;
277        }
278        match self.open_stack[self.open_depth - 1] {
279            b'{' => OpenKind::Message(b'}'),
280            b'<' => OpenKind::Message(b'>'),
281            b'[' => OpenKind::List { scalars: true },
282            MESSAGE_LIST => OpenKind::List { scalars: false },
283            _ => unreachable!("open_stack holds only {{, <, [ and MESSAGE_LIST"),
284        }
285    }
286
287    fn push_open(&mut self, ch: u8) -> Result<(), ParseError> {
288        if self.open_depth >= self.open_stack.len() {
289            return Err(self.err_here(ParseErrorKind::RecursionLimitExceeded));
290        }
291        self.open_stack[self.open_depth] = ch;
292        self.open_depth += 1;
293        Ok(())
294    }
295
296    fn pop_open(&mut self) {
297        debug_assert!(self.open_depth > 0);
298        self.open_depth -= 1;
299    }
300
301    /// Advance `self.cursor` past `n` bytes and then any following
302    /// whitespace or `#`-to-EOL comments.
303    fn consume(&mut self, n: usize) {
304        self.cursor += n;
305        loop {
306            match self.rest().first() {
307                Some(&c) if is_textproto_ws(c) => self.cursor += 1,
308                Some(b'#') => {
309                    // Skip to end of line (or end of input).
310                    let rest = self.rest();
311                    match rest.iter().position(|&b| b == b'\n') {
312                        Some(i) => self.cursor += i + 1,
313                        None => self.cursor = self.input.len(),
314                    }
315                }
316                _ => break,
317            }
318        }
319    }
320
321    /// If the next byte is `c`, consume it (and trailing whitespace) and return true.
322    fn try_consume_char(&mut self, c: u8) -> bool {
323        if self.rest().first() == Some(&c) {
324            self.consume(1);
325            true
326        } else {
327            false
328        }
329    }
330
331    /// Core state machine: produce the next token based on `self.last_kind`
332    /// and the current nesting. Skips over comma/semicolon separators.
333    ///
334    /// Reference: protobuf-go `decode.go` `parseNext`.
335    fn parse_next(&mut self) -> Result<Token<'a>, ParseError> {
336        // Outer loop so we can re-dispatch after consuming a comma/semicolon.
337        loop {
338            self.consume(0);
339            let at_eof = self.rest().is_empty();
340            let open = self.current_open();
341
342            match self.last_kind {
343                LastKind::Bof => {
344                    // Top-level: expect EOF or a field name.
345                    if at_eof {
346                        return Ok(self.emit_eof());
347                    }
348                    return self.parse_field_name();
349                }
350
351                LastKind::Name(has_separator) => {
352                    // After a name: MessageOpen, ListOpen, or Scalar.
353                    if at_eof {
354                        return Err(self.err_here(ParseErrorKind::UnexpectedEof));
355                    }
356                    let ch = self.rest()[0];
357                    match ch {
358                        b'{' | b'<' => {
359                            self.push_open(ch)?;
360                            return Ok(self.emit(TokenKind::MessageOpen, 1));
361                        }
362                        b'[' => {
363                            self.push_open(if has_separator { ch } else { MESSAGE_LIST })?;
364                            return Ok(self.emit(TokenKind::ListOpen, 1));
365                        }
366                        _ if !has_separator => {
367                            return Err(self.err_here(ParseErrorKind::UnexpectedToken {
368                                expected: COLON_BEFORE_SCALAR,
369                            }));
370                        }
371                        _ => return self.parse_scalar(),
372                    }
373                }
374
375                LastKind::Scalar | LastKind::MessageClose | LastKind::ListClose => {
376                    // After a value: either close the current container,
377                    // see a separator, or start the next field name.
378                    match open {
379                        OpenKind::Top => {
380                            if at_eof {
381                                return Ok(self.emit_eof());
382                            }
383                            match self.rest()[0] {
384                                b',' => {
385                                    self.consume(1);
386                                    self.last_kind = LastKind::Comma;
387                                    continue;
388                                }
389                                b';' => {
390                                    self.consume(1);
391                                    self.last_kind = LastKind::Semicolon;
392                                    continue;
393                                }
394                                _ => return self.parse_field_name(),
395                            }
396                        }
397                        OpenKind::Message(close) => {
398                            if at_eof {
399                                return Err(self.err_here(ParseErrorKind::UnexpectedEof));
400                            }
401                            let ch = self.rest()[0];
402                            if ch == close {
403                                self.pop_open();
404                                return Ok(self.emit(TokenKind::MessageClose, 1));
405                            }
406                            if ch == other_close(close) {
407                                return Err(self.err_here(ParseErrorKind::DelimiterMismatch));
408                            }
409                            match ch {
410                                b',' => {
411                                    self.consume(1);
412                                    self.last_kind = LastKind::Comma;
413                                    continue;
414                                }
415                                b';' => {
416                                    self.consume(1);
417                                    self.last_kind = LastKind::Semicolon;
418                                    continue;
419                                }
420                                _ => return self.parse_field_name(),
421                            }
422                        }
423                        OpenKind::List { .. } => {
424                            if at_eof {
425                                return Err(self.err_here(ParseErrorKind::UnexpectedEof));
426                            }
427                            let ch = self.rest()[0];
428                            match ch {
429                                b']' => {
430                                    self.pop_open();
431                                    return Ok(self.emit(TokenKind::ListClose, 1));
432                                }
433                                b',' => {
434                                    self.consume(1);
435                                    self.last_kind = LastKind::Comma;
436                                    continue;
437                                }
438                                _ => {
439                                    return Err(self.err_here(ParseErrorKind::UnexpectedToken {
440                                        expected: "',' or ']'",
441                                    }));
442                                }
443                            }
444                        }
445                    }
446                }
447
448                LastKind::MessageOpen => {
449                    // After `{` or `<`: MessageClose (empty message) or Name.
450                    if at_eof {
451                        return Err(self.err_here(ParseErrorKind::UnexpectedEof));
452                    }
453                    let OpenKind::Message(close) = open else {
454                        unreachable!("MessageOpen always pushes a Message frame")
455                    };
456                    let ch = self.rest()[0];
457                    if ch == close {
458                        self.pop_open();
459                        return Ok(self.emit(TokenKind::MessageClose, 1));
460                    }
461                    if ch == other_close(close) {
462                        return Err(self.err_here(ParseErrorKind::DelimiterMismatch));
463                    }
464                    return self.parse_field_name();
465                }
466
467                LastKind::ListOpen => {
468                    // After `[`: ListClose (empty list), MessageOpen, or Scalar.
469                    if at_eof {
470                        return Err(self.err_here(ParseErrorKind::UnexpectedEof));
471                    }
472                    let OpenKind::List { scalars } = open else {
473                        unreachable!("ListOpen always pushes a List frame")
474                    };
475                    let ch = self.rest()[0];
476                    match ch {
477                        b']' => {
478                            self.pop_open();
479                            return Ok(self.emit(TokenKind::ListClose, 1));
480                        }
481                        b'{' | b'<' => {
482                            self.push_open(ch)?;
483                            return Ok(self.emit(TokenKind::MessageOpen, 1));
484                        }
485                        _ => return self.parse_list_scalar(scalars),
486                    }
487                }
488
489                LastKind::Comma | LastKind::Semicolon => {
490                    // After a separator: close or next name/value.
491                    match open {
492                        OpenKind::Top => {
493                            if at_eof {
494                                return Ok(self.emit_eof());
495                            }
496                            return self.parse_field_name();
497                        }
498                        OpenKind::Message(close) => {
499                            if at_eof {
500                                return Err(self.err_here(ParseErrorKind::UnexpectedEof));
501                            }
502                            let ch = self.rest()[0];
503                            if ch == close {
504                                self.pop_open();
505                                return Ok(self.emit(TokenKind::MessageClose, 1));
506                            }
507                            if ch == other_close(close) {
508                                return Err(self.err_here(ParseErrorKind::DelimiterMismatch));
509                            }
510                            return self.parse_field_name();
511                        }
512                        OpenKind::List { scalars } => {
513                            // Semicolon inside a list is unreachable by
514                            // construction (the Scalar→List arm doesn't
515                            // emit semicolons). Comma: next element.
516                            if at_eof {
517                                return Err(self.err_here(ParseErrorKind::UnexpectedEof));
518                            }
519                            let ch = self.rest()[0];
520                            match ch {
521                                b'{' | b'<' => {
522                                    self.push_open(ch)?;
523                                    return Ok(self.emit(TokenKind::MessageOpen, 1));
524                                }
525                                _ => return self.parse_list_scalar(scalars),
526                            }
527                        }
528                    }
529                }
530            }
531        }
532    }
533
534    /// Parse a scalar element of a list. With `scalars` false (see
535    /// [`OpenKind::List`]) a well-formed scalar is an error at its first byte;
536    /// a malformed one keeps the error `parse_scalar` gives it.
537    fn parse_list_scalar(&mut self, scalars: bool) -> Result<Token<'a>, ParseError> {
538        if scalars {
539            return self.parse_scalar();
540        }
541        let missing_colon = self.err_here(ParseErrorKind::UnexpectedToken {
542            expected: COLON_BEFORE_SCALAR,
543        });
544        // Lexing the scalar moves the cursor; put it back so that a second
545        // call returns the same error.
546        let (cursor, last_kind) = (self.cursor, self.last_kind);
547        self.parse_scalar()?;
548        self.cursor = cursor;
549        self.last_kind = last_kind;
550        Err(missing_colon)
551    }
552
553    fn emit(&mut self, kind: TokenKind, len: usize) -> Token<'a> {
554        let pos = self.cursor;
555        let raw = &self.input[pos..pos + len];
556        self.consume(len);
557        self.last_kind = match kind {
558            TokenKind::Name => LastKind::Name(false),
559            TokenKind::Scalar => LastKind::Scalar,
560            TokenKind::MessageOpen => LastKind::MessageOpen,
561            TokenKind::MessageClose => LastKind::MessageClose,
562            TokenKind::ListOpen => LastKind::ListOpen,
563            TokenKind::ListClose => LastKind::ListClose,
564            TokenKind::Eof => LastKind::Bof, // unused; emit_eof handles Eof
565        };
566        Token {
567            kind,
568            raw,
569            pos,
570            name_kind: NameKind::Ident,
571            scalar_kind: ScalarKind::Number,
572            has_separator: false,
573        }
574    }
575
576    fn emit_eof(&mut self) -> Token<'a> {
577        Token {
578            kind: TokenKind::Eof,
579            raw: &self.input[self.input.len()..],
580            pos: self.input.len(),
581            name_kind: NameKind::Ident,
582            scalar_kind: ScalarKind::Number,
583            has_separator: false,
584        }
585    }
586
587    /// Parse a field name: identifier, `[type.name]`, or field number.
588    /// Also consumes a trailing `:` if present, recording it in
589    /// `has_separator`.
590    fn parse_field_name(&mut self) -> Result<Token<'a>, ParseError> {
591        let start = self.cursor;
592        let rest = self.rest();
593
594        // Bracketed type name: extension or Any URL.
595        if rest[0] == b'[' {
596            // Scan to the matching `]`, skipping `# ...` comments so a `]`
597            // inside one does not close the name. `raw` keeps the trivia.
598            let mut i = 1;
599            while i < rest.len() {
600                match rest[i] {
601                    b']' => break,
602                    b'#' => {
603                        while i < rest.len() && rest[i] != b'\n' {
604                            i += 1;
605                        }
606                    }
607                    _ => i += 1,
608                }
609            }
610            if i >= rest.len() {
611                return Err(self.err(start, ParseErrorKind::UnexpectedEof));
612            }
613            let len = i + 1; // include `]`
614            let raw = &self.input[start..start + len];
615            self.consume(len);
616            let has_separator = self.try_consume_char(b':');
617            self.last_kind = LastKind::Name(has_separator);
618            return Ok(Token {
619                kind: TokenKind::Name,
620                raw,
621                pos: start,
622                name_kind: NameKind::TypeName,
623                scalar_kind: ScalarKind::Number,
624                has_separator,
625            });
626        }
627
628        // Plain identifier.
629        let ilen = parse_ident(rest, false);
630        if ilen > 0 {
631            let raw = &self.input[start..start + ilen];
632            self.consume(ilen);
633            let has_separator = self.try_consume_char(b':');
634            self.last_kind = LastKind::Name(has_separator);
635            return Ok(Token {
636                kind: TokenKind::Name,
637                raw,
638                pos: start,
639                name_kind: NameKind::Ident,
640                scalar_kind: ScalarKind::Number,
641                has_separator,
642            });
643        }
644
645        // Decimal field number.
646        if let Some(num) = lex_number(rest) {
647            if !num.neg && num.kind == NumKind::Dec {
648                // Validate it fits in i32 (field numbers are 29-bit, but
649                // protobuf-go checks i32 range).
650                let s = &self.input[start..start + num.len];
651                if s.parse::<i32>().is_ok() {
652                    let raw = s;
653                    self.consume(num.len);
654                    let has_separator = self.try_consume_char(b':');
655                    self.last_kind = LastKind::Name(has_separator);
656                    return Ok(Token {
657                        kind: TokenKind::Name,
658                        raw,
659                        pos: start,
660                        name_kind: NameKind::FieldNumber,
661                        scalar_kind: ScalarKind::Number,
662                        has_separator,
663                    });
664                }
665            }
666        }
667
668        Err(self.err(
669            start,
670            ParseErrorKind::UnexpectedToken {
671                expected: "field name",
672            },
673        ))
674    }
675
676    /// Parse a scalar: string, literal identifier, or number.
677    fn parse_scalar(&mut self) -> Result<Token<'a>, ParseError> {
678        let start = self.cursor;
679        let rest = self.rest();
680        let first = rest[0];
681
682        // String literal (possibly adjacent-concatenated).
683        if first == b'"' || first == b'\'' {
684            let len = lex_string_run(rest).ok_or_else(|| {
685                self.err(start, ParseErrorKind::InvalidString("unterminated string"))
686            })?;
687            let raw = &self.input[start..start + len];
688            self.consume(len);
689            self.last_kind = LastKind::Scalar;
690            return Ok(Token {
691                kind: TokenKind::Scalar,
692                raw,
693                pos: start,
694                name_kind: NameKind::Ident,
695                scalar_kind: ScalarKind::String,
696                has_separator: false,
697            });
698        }
699
700        // Literal identifier (enum variant, true/false, -inf, etc.).
701        let ilen = parse_ident(rest, true);
702        if ilen > 0 {
703            let raw = &self.input[start..start + ilen];
704            self.consume(ilen);
705            self.last_kind = LastKind::Scalar;
706            return Ok(Token {
707                kind: TokenKind::Scalar,
708                raw,
709                pos: start,
710                name_kind: NameKind::Ident,
711                scalar_kind: ScalarKind::Literal,
712                has_separator: false,
713            });
714        }
715
716        // Number.
717        if let Some(num) = lex_number(rest) {
718            let raw = &self.input[start..start + num.len];
719            self.consume(num.len);
720            self.last_kind = LastKind::Scalar;
721            return Ok(Token {
722                kind: TokenKind::Scalar,
723                raw,
724                pos: start,
725                name_kind: NameKind::Ident,
726                scalar_kind: ScalarKind::Number,
727                has_separator: false,
728            });
729        }
730
731        Err(self.err(
732            start,
733            ParseErrorKind::UnexpectedToken {
734                expected: "scalar value",
735            },
736        ))
737    }
738}
739
740// ── lexing helpers (free functions, no tokenizer state) ──────────────────────
741
742/// The textproto close character for the *other* delimiter style.
743/// `}` ↔ `>`.
744#[inline]
745fn other_close(c: u8) -> u8 {
746    match c {
747        b'}' => b'>',
748        b'>' => b'}',
749        _ => unreachable!(),
750    }
751}
752
753/// Is `c` a token-delimiting byte? (Negation of the identifier/number
754/// character set.)
755#[inline]
756fn is_delim(c: u8) -> bool {
757    !(c == b'-' || c == b'+' || c == b'.' || c == b'_' || c.is_ascii_alphanumeric())
758}
759
760/// Is `c` a textproto whitespace byte?
761///
762/// Per <https://protobuf.dev/reference/protobuf/textformat-spec/#whitespace>
763/// this includes vertical tab and form feed, which `u8::is_ascii_whitespace`
764/// and the common `matches!(c, b' ' | b'\t' | b'\r' | b'\n')` idiom miss.
765#[inline]
766pub(super) const fn is_textproto_ws(c: u8) -> bool {
767    matches!(c, b' ' | b'\t' | b'\r' | b'\n' | b'\x0B' | b'\x0C')
768}
769
770/// Parse an identifier `[_a-zA-Z][_a-zA-Z0-9]*`, optionally with a leading
771/// `-`. Returns the byte length, or 0 if no identifier is present.
772///
773/// Reference: protobuf-go `parseIdent`.
774fn parse_ident(s: &[u8], allow_neg: bool) -> usize {
775    let mut i = 0;
776    if allow_neg && s.first() == Some(&b'-') {
777        i = 1;
778        // The grammar permits whitespace between the sign and the literal
779        // (`FLOAT = [ "-" ] , FLOAT_LIT` where `,` means "may be separated
780        // by whitespace"), so `- inf` is as valid as `-inf`. Matches what
781        // `lex_number` does for `- 42`. The decoder side trims after
782        // stripping `-` so the raw span's internal whitespace is harmless.
783        let after = consume_ws(&s[1..]);
784        i += s.len() - 1 - after.len();
785        if s.get(i).is_none() {
786            return 0;
787        }
788    }
789    match s.get(i) {
790        Some(b'_') | Some(b'a'..=b'z') | Some(b'A'..=b'Z') => i += 1,
791        _ => return 0,
792    }
793    while let Some(&c) = s.get(i) {
794        if c == b'_' || c.is_ascii_alphanumeric() {
795            i += 1;
796        } else {
797            break;
798        }
799    }
800    // Must be followed by a delimiter (or EOF), else this is some glued
801    // nonsense like `foo.bar` in name position without brackets.
802    if let Some(&c) = s.get(i) {
803        if !is_delim(c) {
804            return 0;
805        }
806    }
807    i
808}
809
810/// Numeric literal classification.
811#[derive(Clone, Copy, PartialEq, Eq, Debug)]
812pub(super) enum NumKind {
813    Dec,
814    Hex,
815    Oct,
816    Float,
817}
818
819#[derive(Clone, Copy, Debug)]
820pub(super) struct LexNumber {
821    pub kind: NumKind,
822    pub neg: bool,
823    /// Byte length of the entire lexeme, including `-` and `f` suffix.
824    pub len: usize,
825    /// If `neg`, number of whitespace/comment bytes between `-` and the
826    /// digits. Needed to reconstruct a parseable string.
827    pub sep: usize,
828}
829
830/// Lex a number: decimal, `0x` hex, `0` octal, or float with optional
831/// `e`/`E` exponent and `f`/`F` suffix.
832///
833/// Returns `None` if no number is present or if it's malformed.
834///
835/// Reference: protobuf-go `parseNumber`.
836pub(super) fn lex_number(input: &[u8]) -> Option<LexNumber> {
837    let mut s = input;
838    let mut len = 0;
839    let mut neg = false;
840    let mut sep = 0;
841    let mut kind = NumKind::Dec;
842
843    if s.is_empty() {
844        return None;
845    }
846
847    if s[0] == b'-' {
848        neg = true;
849        s = &s[1..];
850        len += 1;
851        // Consume whitespace/comments between `-` and the digits.
852        let before = s.len();
853        s = consume_ws(s);
854        sep = before - s.len();
855        len += sep;
856        if s.is_empty() {
857            return None;
858        }
859    }
860
861    match s[0] {
862        b'0' => {
863            if s.len() > 1 {
864                match s[1] {
865                    b'x' | b'X' => {
866                        kind = NumKind::Hex;
867                        let mut n = 2;
868                        while s.get(n).is_some_and(|c| c.is_ascii_hexdigit()) {
869                            n += 1;
870                        }
871                        if n == 2 {
872                            return None; // `0x` with no digits
873                        }
874                        len += n;
875                        s = &s[n..];
876                        return finish_number(s, kind, neg, len, sep);
877                    }
878                    b'0'..=b'7' => {
879                        kind = NumKind::Oct;
880                        let mut n = 2;
881                        while s.get(n).is_some_and(|&c| (b'0'..=b'7').contains(&c)) {
882                            n += 1;
883                        }
884                        len += n;
885                        s = &s[n..];
886                        return finish_number(s, kind, neg, len, sep);
887                    }
888                    _ => {}
889                }
890            }
891            s = &s[1..];
892            len += 1;
893        }
894        b'1'..=b'9' => {
895            let mut n = 1;
896            while s.get(n).is_some_and(u8::is_ascii_digit) {
897                n += 1;
898            }
899            s = &s[n..];
900            len += n;
901        }
902        b'.' => {
903            // Leading dot: must have digits after. Flag intent and fall through.
904            kind = NumKind::Float;
905        }
906        _ => return None,
907    }
908
909    // Optional `.` followed by zero-or-more digits.
910    if s.first() == Some(&b'.') {
911        let mut n = 1;
912        // If the leading-dot case brought us here with no digits yet,
913        // require at least one digit after the dot.
914        let had_digits = kind != NumKind::Float;
915        while s.get(n).is_some_and(u8::is_ascii_digit) {
916            n += 1;
917        }
918        if !had_digits && n == 1 {
919            return None; // `.` alone
920        }
921        s = &s[n..];
922        len += n;
923        kind = NumKind::Float;
924    }
925
926    // Optional `e`/`E` [`+`|`-`] digits.
927    if s.len() >= 2 && matches!(s[0], b'e' | b'E') {
928        kind = NumKind::Float;
929        let mut n = 1;
930        if matches!(s[1], b'+' | b'-') {
931            n = 2;
932            if s.len() <= 2 {
933                return None;
934            }
935        }
936        let start = n;
937        while s.get(n).is_some_and(u8::is_ascii_digit) {
938            n += 1;
939        }
940        if n == start {
941            return None; // `e` with no digits
942        }
943        s = &s[n..];
944        len += n;
945    }
946
947    // Optional `f`/`F` suffix.
948    if matches!(s.first(), Some(b'f' | b'F')) {
949        kind = NumKind::Float;
950        s = &s[1..];
951        len += 1;
952    }
953
954    finish_number(s, kind, neg, len, sep)
955}
956
957#[inline]
958fn finish_number(
959    rest: &[u8],
960    kind: NumKind,
961    neg: bool,
962    len: usize,
963    sep: usize,
964) -> Option<LexNumber> {
965    // Must end at a delimiter or EOF.
966    if let Some(&c) = rest.first() {
967        if !is_delim(c) {
968            return None;
969        }
970    }
971    Some(LexNumber {
972        kind,
973        neg,
974        len,
975        sep,
976    })
977}
978
979/// Given a number token's raw text, produce a string suitable for feeding
980/// to Rust's integer/float parsers: strip any `f` suffix and any whitespace
981/// between `-` and the digits.
982pub(super) fn number_for_parse<'a>(raw: &'a str, num: &LexNumber) -> alloc::borrow::Cow<'a, str> {
983    let bytes = raw.as_bytes();
984    let mut end = bytes.len();
985    if num.kind == NumKind::Float && matches!(bytes.last(), Some(b'f' | b'F')) {
986        end -= 1;
987    }
988    if num.neg && num.sep > 0 {
989        // Stitch: `-` + digits (skip the separator run).
990        let mut s = alloc::string::String::with_capacity(end - num.sep);
991        s.push('-');
992        s.push_str(&raw[1 + num.sep..end]);
993        alloc::borrow::Cow::Owned(s)
994    } else {
995        alloc::borrow::Cow::Borrowed(&raw[..end])
996    }
997}
998
999/// Consume leading whitespace and `#`-to-EOL comments.
1000pub(super) fn consume_ws(mut s: &[u8]) -> &[u8] {
1001    loop {
1002        match s.first() {
1003            Some(&c) if is_textproto_ws(c) => s = &s[1..],
1004            Some(b'#') => match s.iter().position(|&b| b == b'\n') {
1005                Some(i) => s = &s[i + 1..],
1006                None => return &[],
1007            },
1008            _ => return s,
1009        }
1010    }
1011}
1012
1013/// Strip the brackets, whitespace and comments from a bracketed field name.
1014/// Returns `None` if `name` is not bracketed.
1015///
1016/// Whitespace and comments may appear anywhere between `[` and `]`, even
1017/// inside an identifier: `[pkg.e xt]` names `pkg.ext`. protobuf-go and C++
1018/// both collect the name characters and discard everything else. A name with
1019/// no trivia stays borrowed.
1020pub(super) fn normalize_bracket_name(name: &str) -> Option<alloc::borrow::Cow<'_, str>> {
1021    let is_trivia_start = |b: u8| is_textproto_ws(b) || b == b'#';
1022    let inner = name.strip_prefix('[')?.strip_suffix(']')?;
1023    if !inner.bytes().any(is_trivia_start) {
1024        return Some(alloc::borrow::Cow::Borrowed(inner));
1025    }
1026
1027    let mut out = alloc::string::String::with_capacity(inner.len());
1028    let mut rest = inner;
1029    while !rest.is_empty() {
1030        // `consume_ws` cuts at 0, after an ASCII byte or at the end of input,
1031        // and `position` finds an ASCII byte, so both cuts are char boundaries.
1032        rest = &rest[rest.len() - consume_ws(rest.as_bytes()).len()..];
1033        let end = rest.bytes().position(is_trivia_start).unwrap_or(rest.len());
1034        out.push_str(&rest[..end]);
1035        rest = &rest[end..];
1036    }
1037    Some(alloc::borrow::Cow::Owned(out))
1038}
1039
1040/// Lex a run of one-or-more adjacent string literals. Returns the total byte
1041/// length including all quotes and any inter-literal whitespace/comments.
1042///
1043/// Escapes are handled only insofar as needed to find the close quote:
1044/// `\"` inside a `"..."` literal does not terminate it.
1045fn lex_string_run(s: &[u8]) -> Option<usize> {
1046    let mut i = 0;
1047    loop {
1048        // One literal.
1049        let quote = *s.get(i)?;
1050        debug_assert!(quote == b'"' || quote == b'\'');
1051        i += 1;
1052        loop {
1053            match s.get(i)? {
1054                &c if c == quote => {
1055                    i += 1;
1056                    break;
1057                }
1058                b'\n' | 0 => return None,
1059                b'\\' => {
1060                    // Skip the escape introducer and the next byte, whatever
1061                    // it is. Validation of the escape happens in `unescape`.
1062                    i += 2;
1063                    if i > s.len() {
1064                        return None;
1065                    }
1066                }
1067                _ => i += 1,
1068            }
1069        }
1070        // After a closing quote: skip inter-literal whitespace/comments and
1071        // check for another opener.
1072        let rest = consume_ws(&s[i..]);
1073        let j = s.len() - rest.len();
1074        if matches!(s.get(j), Some(b'"') | Some(b'\'')) {
1075            i = j;
1076            continue;
1077        }
1078        return Some(i);
1079    }
1080}
1081
1082#[cfg(test)]
1083mod tests {
1084    use super::*;
1085    use alloc::vec::Vec;
1086
1087    /// Drain a tokenizer, collecting (kind, raw) pairs until Eof.
1088    fn drain(input: &str) -> Result<Vec<(TokenKind, &str)>, ParseError> {
1089        let mut t = Tokenizer::new(input);
1090        let mut out = Vec::new();
1091        loop {
1092            let tok = t.read()?;
1093            if tok.kind == TokenKind::Eof {
1094                return Ok(out);
1095            }
1096            out.push((tok.kind, tok.raw));
1097        }
1098    }
1099
1100    // ── whole-input tokenization ────────────────────────────────────────────
1101
1102    #[test]
1103    fn tokenize_simple_field() {
1104        let toks = drain("foo: 42").unwrap();
1105        assert_eq!(toks, [(TokenKind::Name, "foo"), (TokenKind::Scalar, "42")]);
1106    }
1107
1108    #[test]
1109    fn tokenize_nested_message() {
1110        let toks = drain("child { a: 1 b: 2 }").unwrap();
1111        assert_eq!(
1112            toks,
1113            [
1114                (TokenKind::Name, "child"),
1115                (TokenKind::MessageOpen, "{"),
1116                (TokenKind::Name, "a"),
1117                (TokenKind::Scalar, "1"),
1118                (TokenKind::Name, "b"),
1119                (TokenKind::Scalar, "2"),
1120                (TokenKind::MessageClose, "}"),
1121            ]
1122        );
1123    }
1124
1125    #[test]
1126    fn tokenize_angle_delimiters() {
1127        let toks = drain("m < x: 1 >").unwrap();
1128        assert_eq!(toks[1], (TokenKind::MessageOpen, "<"));
1129        assert_eq!(toks[4], (TokenKind::MessageClose, ">"));
1130    }
1131
1132    #[test]
1133    fn tokenize_list() {
1134        let toks = drain("r: [1, 2, 3]").unwrap();
1135        assert_eq!(
1136            toks,
1137            [
1138                (TokenKind::Name, "r"),
1139                (TokenKind::ListOpen, "["),
1140                (TokenKind::Scalar, "1"),
1141                (TokenKind::Scalar, "2"),
1142                (TokenKind::Scalar, "3"),
1143                (TokenKind::ListClose, "]"),
1144            ]
1145        );
1146    }
1147
1148    #[test]
1149    fn tokenize_empty_list() {
1150        let toks = drain("r: []").unwrap();
1151        assert_eq!(
1152            toks,
1153            [
1154                (TokenKind::Name, "r"),
1155                (TokenKind::ListOpen, "["),
1156                (TokenKind::ListClose, "]"),
1157            ]
1158        );
1159    }
1160
1161    #[test]
1162    fn tokenize_message_list() {
1163        // List of messages: `[{...}, {...}]`
1164        let toks = drain("r: [{a: 1}, {a: 2}]").unwrap();
1165        assert_eq!(toks[1], (TokenKind::ListOpen, "["));
1166        assert_eq!(toks[2], (TokenKind::MessageOpen, "{"));
1167        assert_eq!(toks[5], (TokenKind::MessageClose, "}"));
1168        assert_eq!(toks[6], (TokenKind::MessageOpen, "{"));
1169        assert_eq!(toks[9], (TokenKind::MessageClose, "}"));
1170        assert_eq!(toks[10], (TokenKind::ListClose, "]"));
1171    }
1172
1173    #[test]
1174    fn separators_consumed() {
1175        // Comma and semicolon between fields are silently eaten.
1176        let a = drain("a: 1 b: 2").unwrap();
1177        let b = drain("a: 1, b: 2").unwrap();
1178        let c = drain("a: 1; b: 2").unwrap();
1179        assert_eq!(a, b);
1180        assert_eq!(a, c);
1181    }
1182
1183    #[test]
1184    fn comments_skipped() {
1185        let toks = drain("# leading\nfoo: 1 # trailing\nbar: 2").unwrap();
1186        assert_eq!(
1187            toks,
1188            [
1189                (TokenKind::Name, "foo"),
1190                (TokenKind::Scalar, "1"),
1191                (TokenKind::Name, "bar"),
1192                (TokenKind::Scalar, "2"),
1193            ]
1194        );
1195    }
1196
1197    #[test]
1198    fn empty_input() {
1199        assert_eq!(drain("").unwrap(), []);
1200        assert_eq!(drain("  \n\t  ").unwrap(), []);
1201        assert_eq!(drain("# just a comment").unwrap(), []);
1202    }
1203
1204    // ── name variants ───────────────────────────────────────────────────────
1205
1206    #[test]
1207    fn name_kinds() {
1208        #[rustfmt::skip]
1209        let cases: &[(&str, NameKind, &str, bool)] = &[
1210            ("foo: 1",                  NameKind::Ident,       "foo",                  true),
1211            ("foo_bar: 1",              NameKind::Ident,       "foo_bar",              true),
1212            ("_private: 1",             NameKind::Ident,       "_private",             true),
1213            ("msg {}",                  NameKind::Ident,       "msg",                  false), // no colon before message
1214            ("[pkg.ext]: 1",            NameKind::TypeName,    "[pkg.ext]",            true),
1215            ("[a.b.c/d.e]: 1",          NameKind::TypeName,    "[a.b.c/d.e]",          true),
1216            ("[type.googleapis.com/Foo] {}", NameKind::TypeName, "[type.googleapis.com/Foo]", false),
1217            ("42: 1",                   NameKind::FieldNumber, "42",                   true),
1218        ];
1219        for &(input, want_kind, want_raw, want_sep) in cases {
1220            let mut t = Tokenizer::new(input);
1221            let tok = t.read().unwrap();
1222            assert_eq!(tok.kind, TokenKind::Name, "input: {input}");
1223            assert_eq!(tok.name_kind, want_kind, "input: {input}");
1224            assert_eq!(tok.raw, want_raw, "input: {input}");
1225            assert_eq!(tok.has_separator, want_sep, "input: {input}");
1226        }
1227    }
1228
1229    #[test]
1230    fn bracket_name_comment_may_contain_close_bracket() {
1231        let input = "[pkg # ] and é are comment text\n .ext]: 1";
1232        let mut t = Tokenizer::new(input);
1233        let tok = t.read().unwrap();
1234        assert_eq!(tok.kind, TokenKind::Name);
1235        assert_eq!(tok.name_kind, NameKind::TypeName);
1236        assert_eq!(tok.raw, "[pkg # ] and é are comment text\n .ext]");
1237    }
1238
1239    #[test]
1240    fn normalize_bracket_name_trivia() {
1241        assert!(matches!(
1242            normalize_bracket_name("[pkg.ext]"),
1243            Some(alloc::borrow::Cow::Borrowed("pkg.ext"))
1244        ));
1245
1246        #[rustfmt::skip]
1247        let cases = [
1248            ("[ pkg . ext ]", "pkg.ext"),
1249            ("[pkg# comment\n.ext]", "pkg.ext"),
1250            ("[pkg # ] and é are comment text\n . ext]", "pkg.ext"),
1251            ("[type.googleapis.com / pkg . Msg]", "type.googleapis.com/pkg.Msg"),
1252            ("[type.googleapis.com/# comment\npkg.Msg]", "type.googleapis.com/pkg.Msg"),
1253            // Trivia inside an identifier.
1254            ("[pkg.e xt]", "pkg.ext"),
1255            ("[pkg.e# comment\nxt]", "pkg.ext"),
1256            // Non-ASCII characters pass through unchanged.
1257            ("[ pkg.é ]", "pkg.é"),
1258            // A comment with no newline runs to the end of the name.
1259            ("[pkg.ext # comment]", "pkg.ext"),
1260            ("[ # only a comment\n ]", ""),
1261        ];
1262        for (input, want) in cases {
1263            assert_eq!(
1264                normalize_bracket_name(input).as_deref(),
1265                Some(want),
1266                "input: {input:?}"
1267            );
1268        }
1269
1270        for unbracketed in ["pkg.ext", "[pkg.ext", "pkg.ext]", ""] {
1271            assert_eq!(normalize_bracket_name(unbracketed), None);
1272        }
1273    }
1274
1275    #[test]
1276    fn bracket_name_closed_only_inside_a_comment_is_unterminated() {
1277        for input in ["[pkg # ]", "[pkg # ]\n .ext"] {
1278            let err = Tokenizer::new(input).read().unwrap_err();
1279            assert_eq!(err.kind, ParseErrorKind::UnexpectedEof, "input: {input:?}");
1280        }
1281    }
1282
1283    // ── scalar variants ─────────────────────────────────────────────────────
1284
1285    #[test]
1286    fn scalar_kinds() {
1287        #[rustfmt::skip]
1288        let cases: &[(&str, ScalarKind, &str)] = &[
1289            ("f: 42",           ScalarKind::Number,  "42"),
1290            ("f: -7",           ScalarKind::Number,  "-7"),
1291            ("f: 0x1F",         ScalarKind::Number,  "0x1F"),
1292            ("f: 0777",         ScalarKind::Number,  "0777"),
1293            ("f: 1.5",          ScalarKind::Number,  "1.5"),
1294            ("f: 1.5e-3",       ScalarKind::Number,  "1.5e-3"),
1295            ("f: .5",           ScalarKind::Number,  ".5"),
1296            ("f: 1f",           ScalarKind::Number,  "1f"),
1297            ("f: 1.5F",         ScalarKind::Number,  "1.5F"),
1298            (r#"f: "hello""#,   ScalarKind::String,  r#""hello""#),
1299            (r#"f: 'world'"#,   ScalarKind::String,  r#"'world'"#),
1300            (r#"f: "a" "b""#,   ScalarKind::String,  r#""a" "b""#), // adjacent concat
1301            ("f: true",         ScalarKind::Literal, "true"),
1302            ("f: False",        ScalarKind::Literal, "False"),
1303            ("f: FOO_BAR",      ScalarKind::Literal, "FOO_BAR"),    // enum name
1304            ("f: inf",          ScalarKind::Literal, "inf"),
1305            ("f: -inf",         ScalarKind::Literal, "-inf"),       // `-` + ident
1306            ("f: nan",          ScalarKind::Literal, "nan"),
1307        ];
1308        for &(input, want_kind, want_raw) in cases {
1309            let mut t = Tokenizer::new(input);
1310            t.read().unwrap(); // consume name
1311            let tok = t.read().unwrap();
1312            assert_eq!(tok.kind, TokenKind::Scalar, "input: {input}");
1313            assert_eq!(tok.scalar_kind, want_kind, "input: {input}");
1314            assert_eq!(tok.raw, want_raw, "input: {input}");
1315        }
1316    }
1317
1318    #[test]
1319    fn string_escape_not_closing() {
1320        // `\"` inside a string literal must not close it.
1321        let mut t = Tokenizer::new(r#"f: "say \"hi\"""#);
1322        t.read().unwrap();
1323        let tok = t.read().unwrap();
1324        assert_eq!(tok.scalar_kind, ScalarKind::String);
1325        assert_eq!(tok.raw, r#""say \"hi\"""#);
1326    }
1327
1328    #[test]
1329    fn adjacent_strings_no_whitespace() {
1330        let mut t = Tokenizer::new(r#"f: "a"'b'"c""#);
1331        t.read().unwrap();
1332        let tok = t.read().unwrap();
1333        assert_eq!(tok.raw, r#""a"'b'"c""#);
1334    }
1335
1336    #[test]
1337    fn adjacent_strings_with_comment() {
1338        let input = "f: \"a\" # between\n 'b'";
1339        let mut t = Tokenizer::new(input);
1340        t.read().unwrap();
1341        let tok = t.read().unwrap();
1342        assert_eq!(tok.raw, "\"a\" # between\n 'b'");
1343    }
1344
1345    #[test]
1346    fn trailing_comment_is_not_part_of_string_token() {
1347        let mut t = Tokenizer::new("f: \"a\" # \"not a literal\"\ng: 1");
1348        t.read().unwrap();
1349        assert_eq!(t.read().unwrap().raw, "\"a\"");
1350        assert_eq!(t.read().unwrap().raw, "g");
1351    }
1352
1353    #[test]
1354    fn unterminated_literal_after_comment_is_an_error() {
1355        let mut t = Tokenizer::new("f: \"a\" # c\n \"b");
1356        t.read().unwrap();
1357        let err = t.read().unwrap_err();
1358        assert!(matches!(err.kind, ParseErrorKind::InvalidString(_)));
1359    }
1360
1361    // ── number lexing ───────────────────────────────────────────────────────
1362
1363    #[test]
1364    fn lex_number_table() {
1365        // Some((kind, len)) = success, None = not-a-number.
1366        #[rustfmt::skip]
1367        let cases: &[(&str, Option<(NumKind, usize)>)] = &[
1368            ("0",          Some((NumKind::Dec,   1))),
1369            ("42",         Some((NumKind::Dec,   2))),
1370            ("-7",         Some((NumKind::Dec,   2))),
1371            ("- 7",        Some((NumKind::Dec,   3))),  // whitespace after `-` ok
1372            ("0x1F",       Some((NumKind::Hex,   4))),
1373            ("0XFF",       Some((NumKind::Hex,   4))),
1374            ("-0x1",       Some((NumKind::Hex,   4))),
1375            ("0777",       Some((NumKind::Oct,   4))),
1376            ("1.5",        Some((NumKind::Float, 3))),
1377            (".5",         Some((NumKind::Float, 2))),
1378            ("1.",         Some((NumKind::Float, 2))),
1379            ("1e3",        Some((NumKind::Float, 3))),
1380            ("1.5e-3",     Some((NumKind::Float, 6))),
1381            ("1.5E+3",     Some((NumKind::Float, 6))),
1382            ("1f",         Some((NumKind::Float, 2))),
1383            ("1.5F",       Some((NumKind::Float, 4))),
1384            // non-numbers / malformed:
1385            ("",           None),
1386            ("abc",        None),
1387            ("0x",         None),    // no hex digits
1388            (".",          None),    // dot alone
1389            ("1e",         None),    // `e` with no digits
1390            ("1e+",        None),    // `e+` with no digits
1391            ("-",          None),    // bare `-`
1392            ("0x1g",       None),    // glued to non-delim
1393            ("42abc",      None),    // glued to non-delim
1394        ];
1395        for &(input, want) in cases {
1396            let got = lex_number(input.as_bytes()).map(|n| (n.kind, n.len));
1397            assert_eq!(got, want, "input: {input:?}");
1398        }
1399    }
1400
1401    #[test]
1402    fn number_for_parse_strips_suffix() {
1403        let n = lex_number(b"1.5f").unwrap();
1404        assert_eq!(number_for_parse("1.5f", &n), "1.5");
1405    }
1406
1407    #[test]
1408    fn number_for_parse_strips_separator() {
1409        let n = lex_number(b"- 42").unwrap();
1410        assert_eq!(n.sep, 1);
1411        assert_eq!(number_for_parse("- 42", &n), "-42");
1412    }
1413
1414    #[test]
1415    fn number_for_parse_neg_hex_with_sep() {
1416        // `- 0xFF`: both sep-stripping AND hex-prefix-stripping paths.
1417        let n = lex_number(b"- 0xFF").unwrap();
1418        assert_eq!(n.kind, NumKind::Hex);
1419        assert!(n.neg);
1420        assert_eq!(n.sep, 1);
1421        assert_eq!(number_for_parse("- 0xFF", &n), "-0xFF");
1422    }
1423
1424    // ── errors ──────────────────────────────────────────────────────────────
1425
1426    #[test]
1427    fn direct_scalar_requires_colon() {
1428        for name in ["field", "[pkg.ext]", "42"] {
1429            for value in [
1430                "7", "- 7", "0x1F", ".5", "true", "ACTIVE", "-inf", "\"text\"", "'bytes'",
1431            ] {
1432                let input = alloc::format!("{name} {value}");
1433                let mut t = Tokenizer::new(&input);
1434                assert!(!t.read().unwrap().has_separator);
1435                let err = t.read().unwrap_err();
1436                assert_eq!(
1437                    err.kind,
1438                    ParseErrorKind::UnexpectedToken {
1439                        expected: COLON_BEFORE_SCALAR,
1440                    },
1441                    "input: {input:?}"
1442                );
1443                assert_eq!((err.line, err.col), (1, name.len() as u32 + 2));
1444
1445                let valid = alloc::format!("{name}: {value}");
1446                assert!(drain(&valid).is_ok(), "input: {valid:?}");
1447            }
1448        }
1449        for input in ["field\"text\"", "field # comment\n  7", "m { field 7 }"] {
1450            assert!(drain(input).is_err(), "input: {input:?}");
1451        }
1452    }
1453
1454    #[test]
1455    fn scalar_list_requires_colon() {
1456        // (input, column of the first scalar element)
1457        for (input, col) in [
1458            ("f [1]", 4),
1459            ("f [1, 2]", 4),
1460            ("f [\"a\"]", 4),
1461            ("f [ACTIVE]", 4),
1462            ("f [{}, 1]", 8),
1463            ("[pkg.ext] [true]", 12),
1464            ("m { f [1] }", 8),
1465            // The rule belongs to each list: an inner list with a colon
1466            // leaves the outer colon-less one as it was, and a later list
1467            // at the same depth has its own.
1468            ("m [{ f [1] }]", 9),
1469            ("m [{ f: [1] }, 2]", 16),
1470            ("a: [1] b [1]", 11),
1471        ] {
1472            let err = drain(input).unwrap_err();
1473            assert_eq!(
1474                err.kind,
1475                ParseErrorKind::UnexpectedToken {
1476                    expected: COLON_BEFORE_SCALAR,
1477                },
1478                "input: {input:?}"
1479            );
1480            assert_eq!((err.line, err.col), (1, col), "input: {input:?}");
1481        }
1482        // With the colon a list takes scalars, also when it is inside or
1483        // around a colon-less list.
1484        for input in [
1485            "f: [1]",
1486            "f: [1, 2]",
1487            "f: [\"a\"]",
1488            "f: [ACTIVE]",
1489            "f: [{}, 1]",
1490            "[pkg.ext]: [true]",
1491            "m { f: [1] }",
1492            "m [{ f: [1] }, { g: 2 }]",
1493            "m: [{ f [{}] }, 1]",
1494            "a [{}] b: [1]",
1495            "f # comment\n : [1]",
1496        ] {
1497            assert!(drain(input).is_ok(), "input: {input:?}");
1498        }
1499        // A comment between the name and the list changes nothing.
1500        let err = drain("f # comment\n [1]").unwrap_err();
1501        assert_eq!((err.line, err.col), (2, 3));
1502    }
1503
1504    #[test]
1505    fn malformed_element_of_colonless_list_keeps_its_own_error() {
1506        // The trailing comma leaves `]` where an element should start.
1507        let with_colon = drain("m: [{},]").unwrap_err();
1508        let without = drain("m [{},]").unwrap_err();
1509        assert_eq!(without.kind, with_colon.kind);
1510        assert_ne!(
1511            without.kind,
1512            ParseErrorKind::UnexpectedToken {
1513                expected: COLON_BEFORE_SCALAR,
1514            }
1515        );
1516    }
1517
1518    #[test]
1519    fn peeked_colonless_list_open_still_rejects_scalar() {
1520        let mut t = Tokenizer::new("f [1]");
1521        t.read().unwrap();
1522        assert_eq!(t.peek().unwrap().kind, TokenKind::ListOpen);
1523        assert_eq!(t.read().unwrap().kind, TokenKind::ListOpen);
1524        let expected = ParseErrorKind::UnexpectedToken {
1525            expected: COLON_BEFORE_SCALAR,
1526        };
1527        assert_eq!(t.peek().unwrap_err().kind, expected);
1528        assert_eq!(t.read().unwrap_err().kind, expected);
1529    }
1530
1531    #[test]
1532    fn peek_preserves_field_separator() {
1533        for input in ["field: 7", "field 7"] {
1534            let mut t = Tokenizer::new(input);
1535            assert_eq!(t.peek().unwrap().has_separator, input.contains(':'));
1536            assert_eq!(t.peek().unwrap().has_separator, input.contains(':'));
1537            t.read().unwrap();
1538            if input.contains(':') {
1539                assert_eq!(t.peek().unwrap().raw, "7");
1540                assert_eq!(t.read().unwrap().raw, "7");
1541            } else {
1542                assert!(t.peek().is_err());
1543                assert!(t.read().is_err());
1544            }
1545        }
1546    }
1547
1548    #[test]
1549    fn message_values_allow_optional_colon() {
1550        for input in [
1551            "m {}",
1552            "m <i: 7>",
1553            "m: {}",
1554            "m: <i: 7>",
1555            "m [{} , <i: 7>]",
1556            "m: [{} , <i: 7>]",
1557            "m []",
1558            "m: []",
1559            "[pkg.ext] {}",
1560            "[type.googleapis.com/pkg.Msg] {}",
1561        ] {
1562            assert!(drain(input).is_ok(), "input: {input:?}");
1563        }
1564    }
1565
1566    #[test]
1567    fn delimiter_mismatch() {
1568        // Open with `{`, close with `>`.
1569        let err = drain("m { a: 1 >").unwrap_err();
1570        assert_eq!(err.kind, ParseErrorKind::DelimiterMismatch);
1571    }
1572
1573    #[test]
1574    fn delimiter_mismatch_angle() {
1575        let err = drain("m < a: 1 }").unwrap_err();
1576        assert_eq!(err.kind, ParseErrorKind::DelimiterMismatch);
1577    }
1578
1579    #[test]
1580    fn unexpected_eof_in_message() {
1581        let err = drain("m { a: 1").unwrap_err();
1582        assert_eq!(err.kind, ParseErrorKind::UnexpectedEof);
1583    }
1584
1585    #[test]
1586    fn unexpected_eof_in_list() {
1587        let err = drain("r: [1, 2").unwrap_err();
1588        assert_eq!(err.kind, ParseErrorKind::UnexpectedEof);
1589    }
1590
1591    #[test]
1592    fn unterminated_string() {
1593        let err = drain(r#"f: "oops"#).unwrap_err();
1594        assert!(matches!(err.kind, ParseErrorKind::InvalidString(_)));
1595    }
1596
1597    #[test]
1598    fn list_missing_comma() {
1599        // `[1 2]` — missing comma between elements.
1600        let err = drain("r: [1 2]").unwrap_err();
1601        assert!(matches!(
1602            err.kind,
1603            ParseErrorKind::UnexpectedToken {
1604                expected: "',' or ']'"
1605            }
1606        ));
1607    }
1608
1609    #[test]
1610    fn recursion_limit() {
1611        // 101 levels of `m {` should trip the limit.
1612        let depth = crate::message::RECURSION_LIMIT as usize + 1;
1613        let mut s = alloc::string::String::new();
1614        for _ in 0..depth {
1615            s.push_str("m { ");
1616        }
1617        let err = drain(&s).unwrap_err();
1618        assert_eq!(err.kind, ParseErrorKind::RecursionLimitExceeded);
1619    }
1620
1621    // ── peek ────────────────────────────────────────────────────────────────
1622
1623    #[test]
1624    fn peek_does_not_advance() {
1625        let mut t = Tokenizer::new("a: 1 b: 2");
1626        let p1 = t.peek().unwrap();
1627        let p2 = t.peek().unwrap();
1628        assert_eq!(p1.raw, "a");
1629        assert_eq!(p2.raw, "a");
1630        let r = t.read().unwrap();
1631        assert_eq!(r.raw, "a");
1632        let next = t.read().unwrap();
1633        assert_eq!(next.raw, "1");
1634    }
1635
1636    #[test]
1637    fn peek_then_read_preserves_nesting() {
1638        // Peek a MessageOpen, then read it — depth must be correct.
1639        let mut t = Tokenizer::new("m { a: 1 }");
1640        t.read().unwrap(); // name
1641        let p = t.peek().unwrap();
1642        assert_eq!(p.kind, TokenKind::MessageOpen);
1643        t.read().unwrap(); // commit the peeked open
1644                           // Now reading inside the message should work.
1645        assert_eq!(t.read().unwrap().raw, "a");
1646        assert_eq!(t.read().unwrap().raw, "1");
1647        assert_eq!(t.read().unwrap().kind, TokenKind::MessageClose);
1648    }
1649
1650    // ── line/col ────────────────────────────────────────────────────────────
1651
1652    #[test]
1653    fn line_col_table() {
1654        let input = "ab\ncde\nfg";
1655        let t = Tokenizer::new(input);
1656        #[rustfmt::skip]
1657        let cases: &[(usize, (u32, u32))] = &[
1658            (0, (1, 1)),   // 'a'
1659            (1, (1, 2)),   // 'b'
1660            (2, (1, 3)),   // '\n'
1661            (3, (2, 1)),   // 'c'
1662            (5, (2, 3)),   // 'e'
1663            (7, (3, 1)),   // 'f'
1664            (999, (3, 3)), // past end → clamp
1665        ];
1666        for &(pos, want) in cases {
1667            assert_eq!(t.line_col(pos), want, "pos: {pos}");
1668        }
1669    }
1670
1671    #[test]
1672    fn line_col_unicode() {
1673        // 'é' is 2 bytes but 1 column.
1674        let t = Tokenizer::new("éx");
1675        assert_eq!(t.line_col(0), (1, 1));
1676        assert_eq!(t.line_col(1), (1, 1));
1677        assert_eq!(t.line_col(2), (1, 2)); // byte 2 = 'x', column 2
1678    }
1679
1680    #[test]
1681    fn line_col_unicode_interior_offsets_and_eof() {
1682        let t = Tokenizer::new("😀");
1683        for pos in 1..4 {
1684            assert_eq!(t.line_col(pos), (1, 1), "pos: {pos}");
1685        }
1686        assert_eq!(t.line_col(4), (1, 2));
1687        assert_eq!(t.line_col(usize::MAX), (1, 2));
1688
1689        // '€' is 3 bytes; the offsets inside it and the one after it differ.
1690        let t = Tokenizer::new("a€b");
1691        assert_eq!(t.line_col(2), (1, 2));
1692        assert_eq!(t.line_col(3), (1, 2));
1693        assert_eq!(t.line_col(4), (1, 3));
1694    }
1695
1696    #[test]
1697    fn line_col_multiline_unicode_interior_offset() {
1698        let t = Tokenizer::new("é\nx😀y");
1699        let offset_inside_emoji = 6;
1700        assert_eq!(t.line_col(offset_inside_emoji), (2, 2));
1701    }
1702
1703    #[test]
1704    fn error_has_correct_position() {
1705        let err = drain("a: 1\nb: 2\nm { x: 1 >").unwrap_err();
1706        assert_eq!(err.line, 3);
1707        // `>` is at byte offset 9 on line 3 → column 10
1708        assert_eq!(err.col, 10);
1709    }
1710
1711    #[test]
1712    fn bom_is_skipped() {
1713        // UTF-8 BOM (U+FEFF = EF BB BF) at file start should be transparent.
1714        let toks = drain("\u{FEFF}a: 1").unwrap();
1715        assert_eq!(toks, &[(TokenKind::Name, "a"), (TokenKind::Scalar, "1")]);
1716    }
1717
1718    #[test]
1719    fn bom_only_is_empty() {
1720        let toks = drain("\u{FEFF}").unwrap();
1721        assert!(toks.is_empty());
1722    }
1723
1724    #[test]
1725    fn vertical_tab_and_form_feed_are_whitespace() {
1726        // Spec: https://protobuf.dev/reference/protobuf/textformat-spec/#whitespace
1727        let toks = drain("a:\x0B1\x0Cb: 2").unwrap();
1728        assert_eq!(
1729            toks,
1730            &[
1731                (TokenKind::Name, "a"),
1732                (TokenKind::Scalar, "1"),
1733                (TokenKind::Name, "b"),
1734                (TokenKind::Scalar, "2"),
1735            ]
1736        );
1737    }
1738
1739    #[test]
1740    fn signed_literal_with_whitespace() {
1741        // `- inf` is as valid as `-inf` per the grammar. Raw span includes
1742        // the whitespace; decoder handles it.
1743        let toks = drain("f: - inf").unwrap();
1744        assert_eq!(
1745            toks,
1746            &[(TokenKind::Name, "f"), (TokenKind::Scalar, "- inf")]
1747        );
1748    }
1749}