Skip to main content

buffa/text/
decoder.rs

1// Modified by ReallyMe Technologies Limited.
2//! Textproto decoder.
3//!
4//! Wraps a [`Tokenizer`] and adds type-directed value interpretation. A
5//! generated `merge_text` implementation calls
6//! [`read_field_name`](TextDecoder::read_field_name) in a loop, dispatches on
7//! the returned name, and calls the appropriate `read_*` method for that
8//! field's type.
9
10use alloc::borrow::Cow;
11use alloc::vec::Vec;
12
13use super::error::{ParseError, ParseErrorKind};
14use super::string::{unescape, unescape_str, UnescapeError};
15use super::token::{
16    consume_ws, lex_number, normalize_bracket_name, number_for_parse, NumKind, ScalarKind, Token,
17    TokenKind, Tokenizer,
18};
19
20/// Stateful textproto reader.
21///
22/// Drives a [`Tokenizer`] and interprets scalar tokens as the requested Rust
23/// type. Recursion depth is enforced by the tokenizer's fixed-size open-stack
24/// (see [`RECURSION_LIMIT`](crate::RECURSION_LIMIT)), and repeated and map
25/// elements are charged against an element-memory budget defaulting to
26/// [`DEFAULT_ELEMENT_MEMORY_LIMIT`](crate::DEFAULT_ELEMENT_MEMORY_LIMIT)
27/// (see [`with_element_memory_limit`](Self::with_element_memory_limit)).
28pub struct TextDecoder<'a> {
29    tok: Tokenizer<'a>,
30    /// Byte position of the last name returned by `read_field_name`, for
31    /// [`unknown_field`](Self::unknown_field) error reporting.
32    last_name_pos: usize,
33    /// Remaining element-memory budget, shared across the whole parse.
34    ///
35    /// The binary decoder's equivalent rides on `DecodeContext`, which never
36    /// reaches this parser, so textproto carries its own. See
37    /// [`ParseErrorKind::ElementMemoryLimitExceeded`] for the amplification
38    /// it bounds.
39    elem_remaining: usize,
40}
41
42impl<'a> TextDecoder<'a> {
43    /// Create a decoder over `input`.
44    pub fn new(input: &'a str) -> Self {
45        Self {
46            tok: Tokenizer::new(input),
47            last_name_pos: 0,
48            elem_remaining: crate::DEFAULT_ELEMENT_MEMORY_LIMIT,
49        }
50    }
51
52    /// Set the element-memory budget for this parse.
53    ///
54    /// Bounds what the parse *materializes* rather than what it reads; see
55    /// [`ParseErrorKind::ElementMemoryLimitExceeded`]. Defaults to
56    /// [`DEFAULT_ELEMENT_MEMORY_LIMIT`](crate::DEFAULT_ELEMENT_MEMORY_LIMIT);
57    /// pass `usize::MAX` to disable.
58    ///
59    /// The budget is shared across the whole parse, not per field, as
60    /// [`DecodeOptions::with_element_memory_limit`](crate::DecodeOptions::with_element_memory_limit)
61    /// is on the binary side. The two are not charged identically: this
62    /// parser also charges repeated scalars, which the binary decoder
63    /// exempts because a packed 1-byte varint becoming a 4-byte `i32` is not
64    /// an amplification vector. Text has no packed form, so a repeated
65    /// scalar costs at least two input bytes per element and cannot reach a
66    /// worse ratio than that exemption protects — but a textproto scalar
67    /// list above roughly `limit / size_of::<T>()` elements is refused where
68    /// the equivalent binary payload is not.
69    #[must_use]
70    pub fn with_element_memory_limit(mut self, bytes: usize) -> Self {
71        self.elem_remaining = bytes;
72        self
73    }
74
75    /// Charge one element of type `T` against the budget.
76    ///
77    /// Charged before the element is parsed, so an over-budget input costs
78    /// the check rather than the work.
79    ///
80    /// Every repeated element is charged, including scalars — which the
81    /// binary decoder deliberately exempts, on the grounds that a packed
82    /// 1-byte varint becoming a 4-byte `i32` is not an amplification vector
83    /// and bounding it would reject columnar payloads. Text has no packed
84    /// form, so a repeated scalar costs at least two input bytes per element
85    /// and cannot reach a worse ratio than that exemption protects; charging
86    /// it uniformly keeps this one function the whole story rather than
87    /// pushing an element-kind distinction through codegen. The practical
88    /// effect is that a textproto scalar list above roughly
89    /// `DEFAULT_ELEMENT_MEMORY_LIMIT / size_of::<T>()` elements is refused
90    /// where the equivalent binary payload is not.
91    fn charge_element<T>(&mut self) -> Result<(), ParseError> {
92        let cost = core::mem::size_of::<T>();
93        if cost > self.elem_remaining {
94            return Err(self
95                .tok
96                .err_here(ParseErrorKind::ElementMemoryLimitExceeded));
97        }
98        self.elem_remaining -= cost;
99        Ok(())
100    }
101
102    /// Construct a parse error at a token's position.
103    #[inline]
104    fn err_at(&self, tok: &Token<'_>, kind: ParseErrorKind) -> ParseError {
105        let (line, col) = self.tok.line_col(tok.pos);
106        ParseError::new(line, col, kind)
107    }
108
109    /// Read the next field name, or return `None` at end-of-message / EOF.
110    ///
111    /// The returned slice is borrowed directly from the input. For bracketed
112    /// type names (`[pkg.ext]`) it includes the brackets — generated code
113    /// matches against `"[pkg.ext]"` literally.
114    ///
115    /// The `:` after the name is optional before a message value or a list
116    /// of message values. This method returns the name either way; for a
117    /// scalar value, or a scalar list element, that follows a name without
118    /// one, the call that reads the value returns the error. An empty list
119    /// `[]` has no element to read, so it parses with or without the colon,
120    /// unless the caller reads it with
121    /// [`read_repeated_scalar_into`](Self::read_repeated_scalar_into), which
122    /// requires the colon.
123    ///
124    /// # Errors
125    ///
126    /// Any tokenizer error — malformed name, delimiter mismatch, etc.
127    pub fn read_field_name(&mut self) -> Result<Option<&'a str>, ParseError> {
128        let tok = self.tok.peek()?;
129        match tok.kind {
130            TokenKind::Eof | TokenKind::MessageClose => Ok(None),
131            TokenKind::Name => {
132                self.tok.read()?;
133                self.last_name_pos = tok.pos;
134                Ok(Some(tok.raw))
135            }
136            _ => Err(self.err_at(
137                &tok,
138                ParseErrorKind::UnexpectedToken {
139                    expected: "field name",
140                },
141            )),
142        }
143    }
144
145    /// Construct an unknown-field error pointing at the last name returned
146    /// by [`read_field_name`](Self::read_field_name).
147    ///
148    /// Generated `merge_text` wildcard arms return this for any name that is
149    /// neither a declared field nor a declared reserved name; reserved names
150    /// are consumed with [`skip_value`](Self::skip_value) instead.
151    pub fn unknown_field(&self) -> ParseError {
152        let (line, col) = self.tok.line_col(self.last_name_pos);
153        ParseError::new(line, col, ParseErrorKind::UnknownField)
154    }
155
156    /// Read a scalar token, checking it has the right kind.
157    fn read_scalar(&mut self, want: ScalarKind) -> Result<Token<'a>, ParseError> {
158        let tok = self.tok.read()?;
159        if tok.kind != TokenKind::Scalar || tok.scalar_kind != want {
160            return Err(self.err_at(
161                &tok,
162                ParseErrorKind::UnexpectedToken {
163                    expected: match want {
164                        ScalarKind::Number => "number",
165                        ScalarKind::String => "string literal",
166                        ScalarKind::Literal => "identifier",
167                    },
168                },
169            ));
170        }
171        Ok(tok)
172    }
173
174    /// Parse a number token's text as an integer. Shared by all integer readers.
175    ///
176    /// Strips hex/oct prefixes and dispatches to the appropriate radix parser.
177    /// Rejects floats and (for unsigned) negatives.
178    fn parse_int<T>(
179        &self,
180        tok: &Token<'_>,
181        signed: bool,
182        from_dec: fn(&str) -> Option<T>,
183        from_radix: fn(&str, u32) -> Option<T>,
184    ) -> Result<T, ParseError> {
185        let num = lex_number(tok.raw.as_bytes())
186            .ok_or_else(|| self.err_at(tok, ParseErrorKind::InvalidNumber))?;
187        if num.kind == NumKind::Float {
188            return Err(self.err_at(tok, ParseErrorKind::InvalidNumber));
189        }
190        if !signed && num.neg {
191            return Err(self.err_at(tok, ParseErrorKind::InvalidNumber));
192        }
193        let cow = number_for_parse(tok.raw, &num);
194        let s: &str = &cow;
195        let parsed = match num.kind {
196            NumKind::Dec => from_dec(s),
197            NumKind::Hex | NumKind::Oct => {
198                // Strip `-` (if any), strip the base prefix, then parse with
199                // the radix. For signed negative hex/oct, re-prepend `-` —
200                // `i32::from_str_radix("-1F", 16)` works.
201                let (neg, rest) = match s.strip_prefix('-') {
202                    Some(r) => (true, r),
203                    None => (false, s),
204                };
205                let (radix, digits) = if num.kind == NumKind::Hex {
206                    let d = rest
207                        .strip_prefix("0x")
208                        .or_else(|| rest.strip_prefix("0X"))
209                        .ok_or_else(|| self.err_at(tok, ParseErrorKind::InvalidNumber))?;
210                    (16, d)
211                } else {
212                    // Oct: lex only flags Oct when there IS a leading 0 and
213                    // at least one more digit.
214                    (8, rest.strip_prefix('0').unwrap_or(rest))
215                };
216                if !neg {
217                    from_radix(digits, radix)
218                } else {
219                    // Stitch `-` + digits for signed radix parse.
220                    let mut tmp = alloc::string::String::with_capacity(1 + digits.len());
221                    tmp.push('-');
222                    tmp.push_str(digits);
223                    from_radix(&tmp, radix)
224                }
225            }
226            NumKind::Float => unreachable!("rejected above"),
227        };
228        parsed.ok_or_else(|| self.err_at(tok, ParseErrorKind::InvalidNumber))
229    }
230
231    /// Read an `i32` value.
232    ///
233    /// Accepts decimal, `0x` hex, `0` octal. Rejects floats and out-of-range.
234    ///
235    /// # Errors
236    ///
237    /// [`ParseErrorKind::InvalidNumber`] if not a valid in-range integer, or
238    /// [`ParseErrorKind::UnexpectedToken`] if the value is not a number at all.
239    pub fn read_i32(&mut self) -> Result<i32, ParseError> {
240        let tok = self.read_scalar(ScalarKind::Number)?;
241        self.parse_int(
242            &tok,
243            true,
244            |s| s.parse().ok(),
245            |s, r| i32::from_str_radix(s, r).ok(),
246        )
247    }
248
249    /// Read an `i64` value.
250    ///
251    /// # Errors
252    ///
253    /// As [`read_i32`](Self::read_i32).
254    pub fn read_i64(&mut self) -> Result<i64, ParseError> {
255        let tok = self.read_scalar(ScalarKind::Number)?;
256        self.parse_int(
257            &tok,
258            true,
259            |s| s.parse().ok(),
260            |s, r| i64::from_str_radix(s, r).ok(),
261        )
262    }
263
264    /// Read a `u32` value. Rejects negatives.
265    ///
266    /// # Errors
267    ///
268    /// As [`read_i32`](Self::read_i32).
269    pub fn read_u32(&mut self) -> Result<u32, ParseError> {
270        let tok = self.read_scalar(ScalarKind::Number)?;
271        self.parse_int(
272            &tok,
273            false,
274            |s| s.parse().ok(),
275            |s, r| u32::from_str_radix(s, r).ok(),
276        )
277    }
278
279    /// Read a `u64` value. Rejects negatives.
280    ///
281    /// # Errors
282    ///
283    /// As [`read_i32`](Self::read_i32).
284    pub fn read_u64(&mut self) -> Result<u64, ParseError> {
285        let tok = self.read_scalar(ScalarKind::Number)?;
286        self.parse_int(
287            &tok,
288            false,
289            |s| s.parse().ok(),
290            |s, r| u64::from_str_radix(s, r).ok(),
291        )
292    }
293
294    /// Read an `f32` value.
295    ///
296    /// Accepts a decimal number, optionally with an `f` suffix, or one of the
297    /// case-insensitive literals `nan`, `inf` and `infinity`. Either can
298    /// follow a `-`, and whitespace or comments can separate the sign from
299    /// what it negates.
300    ///
301    /// A number rounds once, to the nearest value of the type, ties to even.
302    /// Overflow gives ±∞ and underflow gives ±0.
303    ///
304    /// # Errors
305    ///
306    /// [`ParseErrorKind::InvalidNumber`] if the token is a hex or octal
307    /// number, or an identifier other than `nan`, `inf` or `infinity`.
308    /// [`ParseErrorKind::UnexpectedToken`] if the token is a string or is not
309    /// a scalar.
310    pub fn read_f32(&mut self) -> Result<f32, ParseError> {
311        self.read_float(f32::NAN, f32::INFINITY)
312    }
313
314    /// Read an `f64` value. See [`read_f32`](Self::read_f32).
315    ///
316    /// # Errors
317    ///
318    /// As [`read_f32`](Self::read_f32).
319    pub fn read_f64(&mut self) -> Result<f64, ParseError> {
320        self.read_float(f64::NAN, f64::INFINITY)
321    }
322
323    fn read_float<T>(&mut self, nan: T, infinity: T) -> Result<T, ParseError>
324    where
325        T: core::str::FromStr + core::ops::Neg<Output = T> + Copy,
326    {
327        let tok = self.tok.read()?;
328        if tok.kind != TokenKind::Scalar {
329            return Err(self.err_at(&tok, ParseErrorKind::UnexpectedToken { expected: "number" }));
330        }
331        match tok.scalar_kind {
332            ScalarKind::Literal => {
333                // nan, inf, infinity, -inf, -infinity (case-insensitive).
334                // The tokenizer accepts whitespace/comments between `-` and
335                // the literal, so reuse its trivia handling for the raw span.
336                let (neg, lit) = match tok.raw.strip_prefix('-') {
337                    Some(r) => {
338                        // `consume_ws` only stops after an ASCII whitespace
339                        // byte or a comment's `\n`, neither of which can sit
340                        // inside a multi-byte sequence, so the offset is a
341                        // char boundary even when a comment holds non-ASCII.
342                        let trivia_len = r.len() - consume_ws(r.as_bytes()).len();
343                        debug_assert!(r.is_char_boundary(trivia_len));
344                        (true, &r[trivia_len..])
345                    }
346                    None => (false, tok.raw),
347                };
348                let value = if lit.eq_ignore_ascii_case("nan") {
349                    nan
350                } else if lit.eq_ignore_ascii_case("inf") || lit.eq_ignore_ascii_case("infinity") {
351                    infinity
352                } else {
353                    return Err(self.err_at(&tok, ParseErrorKind::InvalidNumber));
354                };
355                Ok(if neg { -value } else { value })
356            }
357            ScalarKind::Number => {
358                let num = lex_number(tok.raw.as_bytes())
359                    .ok_or_else(|| self.err_at(&tok, ParseErrorKind::InvalidNumber))?;
360                match num.kind {
361                    NumKind::Dec | NumKind::Float => {
362                        // Rust's float parse saturates to ±∞ on overflow and
363                        // to ±0.0 on underflow (since 1.55), both of which are
364                        // the behaviours the textproto spec requires. The sign
365                        // is preserved: `"-0"` and `"-1e-400"` parse to -0.0.
366                        //
367                        // The parse goes straight to `T` so that it rounds
368                        // once. Reading an `f32` as an `f64` and narrowing
369                        // rounds twice, and for a decimal just off the
370                        // midpoint of two `f32` values the result is one ulp
371                        // from the nearest `f32`.
372                        number_for_parse(tok.raw, &num)
373                            .parse::<T>()
374                            .map_err(|_| self.err_at(&tok, ParseErrorKind::InvalidNumber))
375                    }
376                    NumKind::Hex | NumKind::Oct => {
377                        // The textproto spec's FLOAT production is base-10
378                        // only; `0x1` and `01` are not float literals. The
379                        // conformance suite explicitly tests rejection.
380                        Err(self.err_at(&tok, ParseErrorKind::InvalidNumber))
381                    }
382                }
383            }
384            ScalarKind::String => {
385                Err(self.err_at(&tok, ParseErrorKind::UnexpectedToken { expected: "number" }))
386            }
387        }
388    }
389
390    /// Read a `bool` value.
391    ///
392    /// Accepts `true`, `True`, `t`, `false`, `False`, `f`, and `0`/`1`
393    /// (in any integer base). These are the exact literals the C++ text
394    /// parser accepts.
395    ///
396    /// # Errors
397    ///
398    /// [`ParseErrorKind::UnexpectedToken`] if the token is not a recognised
399    /// boolean form.
400    pub fn read_bool(&mut self) -> Result<bool, ParseError> {
401        let tok = self.tok.read()?;
402        if tok.kind != TokenKind::Scalar {
403            return Err(self.err_at(
404                &tok,
405                ParseErrorKind::UnexpectedToken {
406                    expected: "boolean",
407                },
408            ));
409        }
410        match tok.scalar_kind {
411            ScalarKind::Literal => match tok.raw {
412                "true" | "True" | "t" => Ok(true),
413                "false" | "False" | "f" => Ok(false),
414                _ => Err(self.err_at(
415                    &tok,
416                    ParseErrorKind::UnexpectedToken {
417                        expected: "boolean",
418                    },
419                )),
420            },
421            ScalarKind::Number => {
422                // 0/1 in any base (00, 0x1, 01 all accepted by C++).
423                let n: u64 = self.parse_int(
424                    &tok,
425                    false,
426                    |s| s.parse().ok(),
427                    |s, r| u64::from_str_radix(s, r).ok(),
428                )?;
429                match n {
430                    0 => Ok(false),
431                    1 => Ok(true),
432                    _ => Err(self.err_at(
433                        &tok,
434                        ParseErrorKind::UnexpectedToken {
435                            expected: "boolean",
436                        },
437                    )),
438                }
439            }
440            ScalarKind::String => Err(self.err_at(
441                &tok,
442                ParseErrorKind::UnexpectedToken {
443                    expected: "boolean",
444                },
445            )),
446        }
447    }
448
449    /// Read a `string` value. Unescapes and UTF-8-validates.
450    ///
451    /// Borrows the input when the token is a single literal with no escapes.
452    ///
453    /// # Errors
454    ///
455    /// [`ParseErrorKind::InvalidString`] for malformed escapes,
456    /// [`ParseErrorKind::InvalidUtf8`] if the unescaped bytes are not valid
457    /// UTF-8.
458    pub fn read_string(&mut self) -> Result<Cow<'a, str>, ParseError> {
459        let tok = self.read_scalar(ScalarKind::String)?;
460        unescape_str(tok.raw).map_err(|e| {
461            self.err_at(
462                &tok,
463                match e {
464                    UnescapeError::InvalidUtf8 => ParseErrorKind::InvalidUtf8,
465                    UnescapeError::BadEscape(why) => ParseErrorKind::InvalidString(why),
466                },
467            )
468        })
469    }
470
471    /// Read a `bytes` value. Unescapes but does not UTF-8-validate.
472    ///
473    /// # Errors
474    ///
475    /// [`ParseErrorKind::InvalidString`] for malformed escapes.
476    pub fn read_bytes(&mut self) -> Result<Vec<u8>, ParseError> {
477        let tok = self.read_scalar(ScalarKind::String)?;
478        unescape(tok.raw).map_err(|e| {
479            // unescape (byte-level) never returns InvalidUtf8.
480            let UnescapeError::BadEscape(why) = e else {
481                unreachable!("unescape is byte-level")
482            };
483            self.err_at(&tok, ParseErrorKind::InvalidString(why))
484        })
485    }
486
487    /// Read an enum token into `(i32_value, had_known_name, token)`.
488    ///
489    /// Shared by the open and closed enum read methods. For a named variant,
490    /// looks up via [`Enumeration::from_proto_name`](crate::Enumeration::from_proto_name)
491    /// and `had_known_name` is true. For a numeric form, parses the i32 and
492    /// `had_known_name` is false. The caller decides whether unknown numbers
493    /// are acceptable.
494    fn read_enum_inner<E: crate::Enumeration>(
495        &mut self,
496    ) -> Result<(i32, bool, Token<'a>), ParseError> {
497        let tok = self.tok.read()?;
498        if tok.kind != TokenKind::Scalar {
499            return Err(self.err_at(
500                &tok,
501                ParseErrorKind::UnexpectedToken {
502                    expected: "enum value",
503                },
504            ));
505        }
506        match tok.scalar_kind {
507            ScalarKind::Literal if !tok.raw.starts_with('-') => E::from_proto_name(tok.raw)
508                .map(|e| (e.to_i32(), true, tok))
509                .ok_or_else(|| self.err_at(&tok, ParseErrorKind::UnknownEnumValue)),
510            ScalarKind::Number => {
511                let n = self.parse_int(
512                    &tok,
513                    true,
514                    |s| s.parse().ok(),
515                    |s, r| i32::from_str_radix(s, r).ok(),
516                )?;
517                Ok((n, false, tok))
518            }
519            _ => Err(self.err_at(
520                &tok,
521                ParseErrorKind::UnexpectedToken {
522                    expected: "enum value",
523                },
524            )),
525        }
526    }
527
528    /// Read an enum value by variant name or by number (open-enum semantics).
529    ///
530    /// Returns the `i32` wire value. Any in-range integer is accepted — the
531    /// proto3 open-enum model preserves unknown numeric values.
532    ///
533    /// # Errors
534    ///
535    /// [`ParseErrorKind::UnknownEnumValue`] if the name is not a known
536    /// variant; [`ParseErrorKind::InvalidNumber`] if a numeric form is out
537    /// of i32 range.
538    pub fn read_enum_by_name<E: crate::Enumeration>(&mut self) -> Result<i32, ParseError> {
539        self.read_enum_inner::<E>().map(|(n, _, _)| n)
540    }
541
542    /// Read a closed-enum value by variant name or by number.
543    ///
544    /// Returns the enum variant directly. Unknown numeric values are
545    /// **rejected** — the proto2 closed-enum model does not accept values
546    /// outside the defined set in text format. (Binary decode routes them to
547    /// unknown fields; text format has no analogous mechanism, so it errors.)
548    ///
549    /// # Errors
550    ///
551    /// [`ParseErrorKind::UnknownEnumValue`] if the name is not a known
552    /// variant, **or** if a numeric form does not map to a defined variant.
553    pub fn read_closed_enum_by_name<E: crate::Enumeration>(&mut self) -> Result<E, ParseError> {
554        let (n, by_name, tok) = self.read_enum_inner::<E>()?;
555        if by_name {
556            // from_proto_name already succeeded; from_i32 on its to_i32()
557            // is infallible. Avoid a second name lookup.
558            return Ok(E::from_i32(n).expect("from_proto_name returned a valid variant"));
559        }
560        E::from_i32(n).ok_or_else(|| self.err_at(&tok, ParseErrorKind::UnknownEnumValue))
561    }
562
563    /// Enter a `{` or `<`, merge into `msg`, then consume the matching close.
564    ///
565    /// # Errors
566    ///
567    /// Any tokenizer or `merge_text` error, including
568    /// [`ParseErrorKind::RecursionLimitExceeded`] if nesting exceeds
569    /// [`RECURSION_LIMIT`](crate::RECURSION_LIMIT).
570    pub fn merge_message<M: super::TextFormat>(&mut self, msg: &mut M) -> Result<(), ParseError> {
571        self.merge_map_entry(|dec| msg.merge_text(dec))
572    }
573
574    /// Consume `{` or `<`, run `f` over the body, then consume the close.
575    ///
576    /// Closure-taking counterpart to [`merge_message`](Self::merge_message) —
577    /// same rationale as [`TextEncoder::write_map_entry`]. Generated
578    /// `map<K, V>` decode captures `&mut Option<K>` / `&mut Option<V>` and
579    /// dispatches on `"key"` / `"value"` inside the closure without naming a
580    /// concrete entry type.
581    ///
582    /// `#[doc(hidden)]` — codegen support, not public API.
583    ///
584    /// # Errors
585    ///
586    /// Any tokenizer or `f` error.
587    #[doc(hidden)]
588    pub fn merge_map_entry(
589        &mut self,
590        f: impl FnOnce(&mut Self) -> Result<(), ParseError>,
591    ) -> Result<(), ParseError> {
592        let open = self.tok.read()?;
593        if open.kind != TokenKind::MessageOpen {
594            return Err(self.err_at(
595                &open,
596                ParseErrorKind::UnexpectedToken {
597                    expected: "'{' or '<'",
598                },
599            ));
600        }
601        f(self)?;
602        let close = self.tok.read()?;
603        if close.kind != TokenKind::MessageClose {
604            return Err(self.err_at(
605                &close,
606                ParseErrorKind::UnexpectedToken {
607                    expected: "'}' or '>'",
608                },
609            ));
610        }
611        Ok(())
612    }
613
614    /// Read one or more values of a repeated field into `out`.
615    ///
616    /// Handles both forms of a repeated value: the list `f: [1, 2, 3]`
617    /// (consumes `[` and `]`) and the single value `f: 1` (reads exactly one
618    /// element). Generated code calls this once per `f` occurrence; the
619    /// `f: 1 f: 2` form is handled by the outer `read_field_name` loop seeing
620    /// `f` twice.
621    ///
622    /// An empty list `[]` is accepted with or without a `:` after the field
623    /// name, which is the rule for a list of messages. For a field whose
624    /// elements are scalars, use
625    /// [`read_repeated_scalar_into`](Self::read_repeated_scalar_into), which
626    /// requires the colon.
627    ///
628    /// # Errors
629    ///
630    /// Any error from `read_one` or the tokenizer.
631    pub fn read_repeated_into<T>(
632        &mut self,
633        out: &mut Vec<T>,
634        read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
635    ) -> Result<(), ParseError> {
636        self.read_repeated_inner(|value| out.push(value), read_one, false)
637    }
638
639    /// Read one or more values of a repeated scalar field into `out`,
640    /// requiring the `:` after the field name even before an empty list.
641    ///
642    /// Reads the same two forms as
643    /// [`read_repeated_into`](Self::read_repeated_into). Use it when the
644    /// field's elements are numbers, bools, enums, strings or bytes, as
645    /// generated code does. The tokenizer cannot infer the field type from
646    /// `[]`, so this method applies the check for the empty list. For a
647    /// list with elements, or a single value, the tokenizer finds the
648    /// missing colon when this method reads the element.
649    ///
650    /// # Errors
651    ///
652    /// - [`ParseErrorKind::UnexpectedToken`] with `expected` set to
653    ///   `"':' before scalar value"` when the field name has no `:` after
654    ///   it. The error is positioned at the `[` of an empty list, and at the
655    ///   element for a list with elements or a single value.
656    /// - Any other error from `read_one` or the tokenizer.
657    pub fn read_repeated_scalar_into<T>(
658        &mut self,
659        out: &mut Vec<T>,
660        read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
661    ) -> Result<(), ParseError> {
662        self.read_repeated_inner(|value| out.push(value), read_one, true)
663    }
664
665    /// Decode repeated messages directly into a custom collection.
666    /// # Errors
667    /// Returns a tokenizer or element-decoding error, preserving earlier elements.
668    pub fn read_repeated_into_storage<T, C: crate::ProtoList<T>>(
669        &mut self,
670        out: &mut C,
671        read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
672    ) -> Result<(), ParseError> {
673        self.read_repeated_inner(|value| crate::ProtoList::push(out, value), read_one, false)
674    }
675    /// Decode repeated scalars directly into a custom collection, requiring `:`.
676    /// # Errors
677    /// Returns a tokenizer or element-decoding error, preserving earlier elements.
678    pub fn read_repeated_scalar_into_storage<T, C: crate::ProtoList<T>>(
679        &mut self,
680        out: &mut C,
681        read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
682    ) -> Result<(), ParseError> {
683        self.read_repeated_inner(|value| crate::ProtoList::push(out, value), read_one, true)
684    }
685
686    /// Decode repeated message values and pass each one directly to its owner.
687    /// # Errors
688    /// Returns a tokenizer or element-decoding error after committing earlier values.
689    pub fn read_repeated_with<T>(
690        &mut self,
691        push: impl FnMut(T),
692        read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
693    ) -> Result<(), ParseError> {
694        self.read_repeated_inner(push, read_one, false)
695    }
696
697    fn read_repeated_inner<T>(
698        &mut self,
699        mut push: impl FnMut(T),
700        mut read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
701        empty_list_requires_separator: bool,
702    ) -> Result<(), ParseError> {
703        if self.tok.peek()?.kind == TokenKind::ListOpen {
704            let open = self.tok.read()?;
705            if self.tok.peek()?.kind == TokenKind::ListClose {
706                if empty_list_requires_separator && self.tok.in_list_without_separator() {
707                    return Err(self.err_at(
708                        &open,
709                        ParseErrorKind::UnexpectedToken {
710                            expected: super::token::COLON_BEFORE_SCALAR,
711                        },
712                    ));
713                }
714                self.tok.read()?; // empty: `[]`
715                return Ok(());
716            }
717            loop {
718                self.charge_element::<T>()?;
719                push(read_one(self)?);
720                // The tokenizer consumes the comma between elements. When
721                // there are no more elements, `ListClose` is next.
722                if self.tok.peek()?.kind == TokenKind::ListClose {
723                    self.tok.read()?;
724                    return Ok(());
725                }
726            }
727        }
728        // Single-value form.
729        self.charge_element::<T>()?;
730        push(read_one(self)?);
731        Ok(())
732    }
733
734    /// Parse an `Any`-expansion body: the `[type_url] { fields }` form.
735    ///
736    /// `name` is the bracketed name as returned by
737    /// [`read_field_name`](Self::read_field_name), e.g.
738    /// `"[type.googleapis.com/pkg.Foo]"`. The brackets, and any whitespace or
739    /// `#` comments between them, are stripped here and the result is looked
740    /// up in the global text-format `Any` map (installed via
741    /// [`set_type_registry`]); the registered `text_merge` then consumes the
742    /// `{ ... }` body and re-encodes to wire bytes suitable for `Any.value`.
743    ///
744    /// Returns `(type_url, value_bytes)`, where `type_url` is the URL between
745    /// the brackets with whitespace and comments removed. It is the value for
746    /// `Any.type_url`, and is borrowed from `name` unless whitespace or
747    /// comments were removed. Its prefix can differ from the registered
748    /// URL's, because the map matches an unregistered URL by the message full
749    /// name after its last `/`.
750    ///
751    /// # Errors
752    ///
753    /// [`ParseErrorKind::UnknownField`] if neither the URL nor its message
754    /// full name is registered — this matches `AnyFieldWithInvalidType` in
755    /// the conformance suite, which expects parse failure on an unknown URL.
756    ///
757    /// [`set_type_registry`]: crate::type_registry::set_type_registry
758    pub fn read_any_expansion(
759        &mut self,
760        name: &'a str,
761    ) -> Result<(Cow<'a, str>, Vec<u8>), ParseError> {
762        // read_field_name only returns bracketed names for NameKind::TypeName;
763        // a missing bracket here means the caller dispatched wrong.
764        let url = normalize_bracket_name(name).ok_or_else(|| self.unknown_field())?;
765        let entry =
766            crate::type_registry::global_text_any(&url).ok_or_else(|| self.unknown_field())?;
767        let bytes = (entry.text_merge)(self)?;
768        Ok((url, bytes))
769    }
770
771    /// Parse an extension bracket body: the `[pkg.ext] { ... }` form.
772    ///
773    /// `name` is the bracketed name as returned by
774    /// [`read_field_name`](Self::read_field_name). The brackets, and any
775    /// whitespace or `#` comments between them, are stripped and the result
776    /// is looked up by `full_name` in the global text-format
777    /// extension map (installed via [`set_type_registry`]); the registered
778    /// `text_merge` consumes the value and produces unknown-field records at
779    /// the extension's field number.
780    ///
781    /// Repeated message and group entries accept a message list or a single
782    /// message value and produce one record per element.
783    ///
784    /// # Errors
785    ///
786    /// [`ParseErrorKind::UnknownField`] if the name is not registered or the
787    /// registered entry extends a different message. Strict by default —
788    /// protobuf-go's `prototext` behaviour, and what the
789    /// `GroupFieldExtensionGroupName` conformance test expects.
790    ///
791    /// [`set_type_registry`]: crate::type_registry::set_type_registry
792    pub fn read_extension(
793        &mut self,
794        name: &str,
795        extendee: &str,
796    ) -> Result<Vec<crate::UnknownField>, ParseError> {
797        let full = normalize_bracket_name(name).ok_or_else(|| self.unknown_field())?;
798        let Some(entry) = crate::type_registry::global_text_ext_by_name(full.as_ref()) else {
799            return Err(self.unknown_field());
800        };
801        if entry.extendee != extendee {
802            return Err(self.unknown_field());
803        }
804        (entry.text_merge)(self, entry.number)
805    }
806
807    /// Consume a field's value without interpreting it.
808    ///
809    /// Generated `merge_text` impls call this for declared reserved field
810    /// names; hand-written impls can call it for any name they want to accept
811    /// and discard rather than fail on. Handles scalars, messages (`{}` and
812    /// `<>`), and lists (recursively).
813    ///
814    /// # Errors
815    ///
816    /// Any tokenizer error in the skipped span.
817    pub fn skip_value(&mut self) -> Result<(), ParseError> {
818        let tok = self.tok.peek()?;
819        match tok.kind {
820            TokenKind::Scalar => {
821                self.tok.read()?;
822                Ok(())
823            }
824            TokenKind::MessageOpen => {
825                self.tok.read()?;
826                // read_field_name returns None at MessageClose — loop until then.
827                while self.read_field_name()?.is_some() {
828                    self.skip_value()?;
829                }
830                // Consume the MessageClose.
831                let close = self.tok.read()?;
832                debug_assert_eq!(close.kind, TokenKind::MessageClose);
833                Ok(())
834            }
835            TokenKind::ListOpen => {
836                self.tok.read()?;
837                loop {
838                    if self.tok.peek()?.kind == TokenKind::ListClose {
839                        self.tok.read()?;
840                        return Ok(());
841                    }
842                    self.skip_value()?;
843                }
844            }
845            _ => Err(self.err_at(&tok, ParseErrorKind::UnexpectedToken { expected: "value" })),
846        }
847    }
848}
849
850// ─────────────────────────────────────────────────────────────────────────────
851// Tests
852// ─────────────────────────────────────────────────────────────────────────────
853
854#[cfg(test)]
855mod tests {
856    use super::super::encoder::TextEncoder;
857    use super::super::{decode_from_str, encode_to_string, encode_to_string_pretty, TextFormat};
858    use super::*;
859    use alloc::boxed::Box;
860    use alloc::string::String;
861    use alloc::vec;
862
863    // ── test enum ───────────────────────────────────────────────────────────
864
865    #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
866    enum Color {
867        Red,
868        Green,
869        Blue,
870    }
871
872    impl crate::Enumeration for Color {
873        fn from_i32(v: i32) -> Option<Self> {
874            match v {
875                0 => Some(Color::Red),
876                1 => Some(Color::Green),
877                2 => Some(Color::Blue),
878                _ => None,
879            }
880        }
881        fn to_i32(&self) -> i32 {
882            match self {
883                Color::Red => 0,
884                Color::Green => 1,
885                Color::Blue => 2,
886            }
887        }
888        fn proto_name(&self) -> &'static str {
889            match self {
890                Color::Red => "RED",
891                Color::Green => "GREEN",
892                Color::Blue => "BLUE",
893            }
894        }
895        fn from_proto_name(name: &str) -> Option<Self> {
896            match name {
897                "RED" => Some(Color::Red),
898                "GREEN" => Some(Color::Green),
899                "BLUE" => Some(Color::Blue),
900                _ => None,
901            }
902        }
903    }
904
905    // ── test message ────────────────────────────────────────────────────────
906    //
907    // Hand-implemented to avoid depending on codegen in the runtime crate's
908    // own tests. Mirrors the pattern in `extension.rs` tests.
909
910    #[derive(Default, Clone, PartialEq, Debug)]
911    struct TestMsg {
912        i: i32,
913        s: String,
914        items: Vec<i64>,
915        child: Option<Box<TestMsg>>,
916    }
917
918    impl crate::DefaultInstance for TestMsg {
919        fn default_instance() -> &'static Self {
920            static INST: crate::__private::OnceBox<TestMsg> = crate::__private::OnceBox::new();
921            INST.get_or_init(|| Box::new(TestMsg::default()))
922        }
923    }
924
925    impl crate::Message for TestMsg {
926        fn compute_size(&self, _cache: &mut crate::SizeCache) -> u32 {
927            0
928        }
929        fn write_to(&self, _cache: &mut crate::SizeCache, _buf: &mut impl crate::EncodeSink) {}
930        fn merge_field(
931            &mut self,
932            tag: crate::encoding::Tag,
933            buf: &mut impl bytes::Buf,
934            _ctx: crate::DecodeContext<'_>,
935        ) -> Result<(), crate::DecodeError> {
936            crate::encoding::skip_field(tag, buf)
937        }
938        fn clear(&mut self) {
939            *self = Self::default();
940        }
941    }
942
943    impl TextFormat for TestMsg {
944        fn encode_text(&self, enc: &mut TextEncoder<'_>) -> core::fmt::Result {
945            if self.i != 0 {
946                enc.write_field_name("i")?;
947                enc.write_i32(self.i)?;
948            }
949            if !self.s.is_empty() {
950                enc.write_field_name("s")?;
951                enc.write_string(&self.s)?;
952            }
953            for &item in &self.items {
954                enc.write_field_name("items")?;
955                enc.write_i64(item)?;
956            }
957            if let Some(child) = &self.child {
958                enc.write_field_name("child")?;
959                enc.write_message(child.as_ref())?;
960            }
961            Ok(())
962        }
963
964        fn merge_text(&mut self, dec: &mut TextDecoder<'_>) -> Result<(), ParseError> {
965            while let Some(name) = dec.read_field_name()? {
966                match name {
967                    "i" => self.i = dec.read_i32()?,
968                    "s" => self.s = dec.read_string()?.into_owned(),
969                    "items" => dec.read_repeated_scalar_into(&mut self.items, |d| d.read_i64())?,
970                    "child" => {
971                        let child = self.child.get_or_insert_with(Default::default);
972                        dec.merge_message(child.as_mut())?;
973                    }
974                    _ => dec.skip_value()?,
975                }
976            }
977            Ok(())
978        }
979    }
980
981    // ── scalar read tables ──────────────────────────────────────────────────
982
983    #[test]
984    fn read_i32_table() {
985        #[rustfmt::skip]
986        let cases: &[(&str, Option<i32>)] = &[
987            ("42",          Some(42)),
988            ("-7",          Some(-7)),
989            ("0",           Some(0)),
990            ("0x1F",        Some(31)),
991            ("0X1f",        Some(31)),
992            ("-0x10",       Some(-16)),
993            ("0777",        Some(511)),
994            ("-010",        Some(-8)),
995            ("2147483647",  Some(i32::MAX)),
996            ("-2147483648", Some(i32::MIN)),
997            // errors:
998            ("2147483648",  None),  // overflow
999            ("1.5",         None),  // float
1000            ("1f",          None),  // float suffix
1001        ];
1002        for &(input, want) in cases {
1003            // Wrap in a field so the tokenizer is in value-expecting state.
1004            let full = alloc::format!("f: {input}");
1005            let mut d = TextDecoder::new(&full);
1006            d.read_field_name().unwrap();
1007            assert_eq!(d.read_i32().ok(), want, "input: {input}");
1008        }
1009    }
1010
1011    #[test]
1012    fn read_u32_rejects_negative() {
1013        let mut d = TextDecoder::new("f: -1");
1014        d.read_field_name().unwrap();
1015        assert_eq!(d.read_u32().ok(), None);
1016    }
1017
1018    #[test]
1019    fn read_u64_table() {
1020        #[rustfmt::skip]
1021        let cases: &[(&str, Option<u64>)] = &[
1022            ("f: 0",                       Some(0)),
1023            ("f: 18446744073709551615",    Some(u64::MAX)),
1024            ("f: 0xFFFFFFFFFFFFFFFF",      Some(u64::MAX)),
1025            ("f: 18446744073709551616",    None),  // overflow
1026            ("f: -1",                      None),  // negative
1027        ];
1028        for &(input, want) in cases {
1029            let mut d = TextDecoder::new(input);
1030            d.read_field_name().unwrap();
1031            assert_eq!(d.read_u64().ok(), want, "input: {input}");
1032        }
1033    }
1034
1035    #[test]
1036    fn read_f64_table() {
1037        // Floats can't use Option<f64> + == (NaN), so check classes.
1038        #[rustfmt::skip]
1039        let cases: &[(&str, &str)] = &[
1040            ("f: 1.5",        "1.5"),
1041            ("f: -2.5",       "-2.5"),
1042            ("f: .5",         "0.5"),
1043            ("f: 1e3",        "1000"),
1044            ("f: 1.5f",       "1.5"),     // f suffix stripped
1045            ("f: 42",         "42"),      // int → float
1046            ("f: 0",          "0"),       // plain zero (Dec, not Oct)
1047            ("f: 0.0",        "0"),
1048            ("f: 0e0",        "0"),
1049            ("f: inf",        "inf"),
1050            ("f: -inf",       "-inf"),
1051            ("f: - inf",      "-inf"),    // whitespace after sign
1052            ("f: -\tinf",     "-inf"),
1053            ("f: - # c\ninf", "-inf"),  // comment after sign
1054            ("f: -#c\n nan", "nan"),
1055            ("f: infinity",   "inf"),
1056            ("f: Infinity",   "inf"),
1057            ("f: -INFINITY",  "-inf"),
1058            ("f: nan",        "nan"),
1059            ("f: NaN",        "nan"),
1060        ];
1061        for &(input, want) in cases {
1062            let mut d = TextDecoder::new(input);
1063            d.read_field_name().unwrap();
1064            let v = d
1065                .read_f64()
1066                .unwrap_or_else(|e| panic!("input {input}: {e}"));
1067            let got = if v.is_nan() {
1068                String::from("nan")
1069            } else if v.is_infinite() {
1070                String::from(if v > 0.0 { "inf" } else { "-inf" })
1071            } else {
1072                alloc::format!("{v}")
1073            };
1074            assert_eq!(got, want, "input: {input}");
1075        }
1076    }
1077
1078    #[test]
1079    fn read_f64_negative_zero() {
1080        // -0.0 must be preserved through the parse path. `f64::from_str`
1081        // handles this correctly for `"-0"`, `"-0.0"`, and underflow like
1082        // `"-1e-400"` — the sign bit survives. This test guards against
1083        // any future integer-parse-then-cast shortcut that would lose it.
1084        //
1085        // `-0.0 == 0.0` is true in IEEE float comparison, so assertions
1086        // must check the bit pattern (or sign) explicitly.
1087        #[rustfmt::skip]
1088        let cases: &[&str] = &[
1089            "-0",        // NumKind::Dec
1090            "-0.0",      // NumKind::Float
1091            "-0f",       // NumKind::Float, f suffix
1092            "-0F",       // NumKind::Float, F suffix
1093            "-0.0f",
1094            "-0e0",      // NumKind::Float, exponent form
1095            "-1e-400",   // underflow → -0.0
1096            "- 0",       // whitespace between sign and digits
1097        ];
1098        for &input in cases {
1099            let full = alloc::format!("f: {input}");
1100            let mut d = TextDecoder::new(&full);
1101            d.read_field_name().unwrap();
1102            let v = d.read_f64().unwrap_or_else(|e| panic!("{input}: {e}"));
1103            assert!(
1104                v == 0.0 && v.is_sign_negative(),
1105                "{input}: want -0.0, got {v:?} (bits {:#018x})",
1106                v.to_bits()
1107            );
1108        }
1109    }
1110
1111    #[test]
1112    fn read_f32_negative_zero() {
1113        let mut d = TextDecoder::new("f: -0");
1114        d.read_field_name().unwrap();
1115        let v = d.read_f32().unwrap();
1116        assert!(v == 0.0 && v.is_sign_negative());
1117        assert_eq!(v.to_bits(), (-0.0f32).to_bits());
1118    }
1119
1120    #[test]
1121    fn read_f32_rounds_once() {
1122        // Each input rounds to a different `f32` when it is read as an `f64`
1123        // and narrowed: the `f64` lands on the midpoint of two `f32` values,
1124        // and the tie then goes the other way.
1125        let cases = [
1126            // Just past the midpoint of 1.0 and its neighbour.
1127            ("1.0000000596046447753906250000000000000001", 0x3f80_0001u32),
1128            (
1129                "-1.0000000596046447753906250000000000000001",
1130                0xbf80_0001u32,
1131            ),
1132            // The shortest decimal that identifies this `f32`.
1133            ("7.038531e-26", 0x15ae_43fd),
1134            // One below 2^128 - 2^103, the midpoint of `f32::MAX` and 2^128:
1135            // narrowed, it overflows to infinity.
1136            ("340282356779733661637539395458142568447", 0x7f7f_ffff),
1137            // Just above 2^-150, the midpoint of zero and the smallest
1138            // subnormal: narrowed, it underflows to zero.
1139            (
1140                "7.006492321624085354618647916449580656401309709382578858785341419448955413429303\
1141                 00743319094181060791015625000000001e-46",
1142                0x0000_0001,
1143            ),
1144        ];
1145        for (input, expected) in cases {
1146            let narrowed = input.parse::<f64>().unwrap() as f32;
1147            assert_ne!(narrowed.to_bits(), expected, "input: {input}");
1148
1149            let full = alloc::format!("f: {input}");
1150            let mut d = TextDecoder::new(&full);
1151            d.read_field_name().unwrap();
1152            let got = d.read_f32().unwrap();
1153            assert_eq!(got.to_bits(), expected, "input: {input}");
1154        }
1155    }
1156
1157    #[test]
1158    fn read_f32_ties_overflow_underflow_and_suffix() {
1159        let cases = [
1160            // The exact midpoint of 1.0 and its neighbour ties to even.
1161            ("1.000000059604644775390625", 1.0f32),
1162            ("1e39", f32::INFINITY),
1163            ("-1e39", f32::NEG_INFINITY),
1164            ("1e-46", 0.0),
1165            ("-1e-46", -0.0),
1166            ("1.5f", 1.5),
1167            ("3.4028235e38", f32::MAX),
1168        ];
1169        for (input, expected) in cases {
1170            let full = alloc::format!("f: {input}");
1171            let mut d = TextDecoder::new(&full);
1172            d.read_field_name().unwrap();
1173            let got = d.read_f32().unwrap();
1174            assert_eq!(got.to_bits(), expected.to_bits(), "input: {input}");
1175        }
1176    }
1177
1178    #[test]
1179    fn read_f64_rejects_hex_and_octal() {
1180        // The textproto FLOAT grammar is base-10 only. Conformance tests
1181        // `FloatFieldNoHex`, `FloatFieldNoOctal` (and their negative forms)
1182        // require rejection.
1183        #[rustfmt::skip]
1184        let cases: &[&str] = &[
1185            "0x1",   "-0x1",   "0xFF",
1186            "01",    "-01",    "0777",
1187        ];
1188        for &input in cases {
1189            let full = alloc::format!("f: {input}");
1190            let mut d = TextDecoder::new(&full);
1191            d.read_field_name().unwrap();
1192            let err = d
1193                .read_f64()
1194                .expect_err(&alloc::format!("{input}: should reject hex/oct for float"));
1195            assert_eq!(err.kind, ParseErrorKind::InvalidNumber, "input: {input}");
1196        }
1197        // Sanity: these zero forms start with `0` but are valid floats
1198        // (Dec or Float, never Oct — lex_number only flags Oct when a
1199        // second octal digit follows).
1200        for &input in &["0", "0.0", "0e0", "0.", "0f"] {
1201            let full = alloc::format!("f: {input}");
1202            let mut d = TextDecoder::new(&full);
1203            d.read_field_name().unwrap();
1204            assert!(d.read_f64().is_ok(), "{input}: should be accepted");
1205        }
1206    }
1207
1208    #[test]
1209    fn read_f64_rejects_non_float_literal() {
1210        let mut d = TextDecoder::new("f: hello");
1211        d.read_field_name().unwrap();
1212        assert!(d.read_f64().is_err());
1213    }
1214
1215    #[test]
1216    fn read_bool_table() {
1217        #[rustfmt::skip]
1218        let cases: &[(&str, Option<bool>)] = &[
1219            ("f: true",   Some(true)),
1220            ("f: True",   Some(true)),
1221            ("f: t",      Some(true)),
1222            ("f: false",  Some(false)),
1223            ("f: False",  Some(false)),
1224            ("f: f",      Some(false)),
1225            ("f: 1",      Some(true)),
1226            ("f: 0",      Some(false)),
1227            ("f: 0x1",    Some(true)),
1228            ("f: 01",     Some(true)),   // octal 1
1229            // errors:
1230            ("f: 2",      None),
1231            ("f: yes",    None),
1232            ("f: TRUE",   None),  // not in C++'s accepted set
1233        ];
1234        for &(input, want) in cases {
1235            let mut d = TextDecoder::new(input);
1236            d.read_field_name().unwrap();
1237            assert_eq!(d.read_bool().ok(), want, "input: {input}");
1238        }
1239    }
1240
1241    #[test]
1242    fn read_string_table() {
1243        #[rustfmt::skip]
1244        let cases: &[(&str, Option<&str>)] = &[
1245            (r#"f: "hello""#,         Some("hello")),
1246            (r#"f: 'world'"#,         Some("world")),
1247            (r#"f: "say \"hi\"""#,    Some("say \"hi\"")),
1248            (r#"f: "foo" "bar""#,     Some("foobar")),
1249            ("f: \"foo\" # c\n \"bar\"", Some("foobar")),
1250            (r#"f: """#,              Some("")),
1251            (r#"f: 42"#,              None),  // not a string
1252            (r#"f: "\xFF""#,          None),  // invalid UTF-8
1253        ];
1254        for &(input, want) in cases {
1255            let mut d = TextDecoder::new(input);
1256            d.read_field_name().unwrap();
1257            let got = d.read_string().ok();
1258            assert_eq!(got.as_deref(), want, "input: {input}");
1259        }
1260    }
1261
1262    #[test]
1263    fn read_bytes_accepts_non_utf8() {
1264        let mut d = TextDecoder::new(r#"f: "\xFF\x00\x01""#);
1265        d.read_field_name().unwrap();
1266        assert_eq!(d.read_bytes().unwrap(), vec![0xFF, 0x00, 0x01]);
1267    }
1268
1269    #[test]
1270    fn read_enum_table() {
1271        #[rustfmt::skip]
1272        let cases: &[(&str, Option<i32>)] = &[
1273            ("f: RED",    Some(0)),
1274            ("f: GREEN",  Some(1)),
1275            ("f: BLUE",   Some(2)),
1276            ("f: 0",      Some(0)),     // numeric form
1277            ("f: 99",     Some(99)),    // unknown number still OK (open enum)
1278            ("f: PURPLE", None),        // unknown name → error
1279            ("f: -RED",   None),        // negative-prefixed name not an enum
1280        ];
1281        for &(input, want) in cases {
1282            let mut d = TextDecoder::new(input);
1283            d.read_field_name().unwrap();
1284            assert_eq!(d.read_enum_by_name::<Color>().ok(), want, "input: {input}");
1285        }
1286    }
1287
1288    #[test]
1289    fn read_enum_unknown_name_error_kind() {
1290        let mut d = TextDecoder::new("f: PURPLE");
1291        d.read_field_name().unwrap();
1292        let err = d.read_enum_by_name::<Color>().unwrap_err();
1293        assert_eq!(err.kind, ParseErrorKind::UnknownEnumValue);
1294    }
1295
1296    #[test]
1297    fn read_closed_enum_table() {
1298        // Closed enums (proto2 semantics) reject unknown NUMERIC values in
1299        // text format. Open enums accept them; see `read_enum_table` above
1300        // where `99` → `Some(99)`.
1301        #[rustfmt::skip]
1302        let cases: &[(&str, Option<Color>)] = &[
1303            ("f: RED",    Some(Color::Red)),
1304            ("f: GREEN",  Some(Color::Green)),
1305            ("f: 0",      Some(Color::Red)),     // numeric → known variant
1306            ("f: 2",      Some(Color::Blue)),
1307            ("f: 99",     None),                 // unknown number → error
1308            ("f: -1",     None),                 // negative → error
1309            ("f: PURPLE", None),                 // unknown name → error (same as open)
1310        ];
1311        for &(input, want) in cases {
1312            let mut d = TextDecoder::new(input);
1313            d.read_field_name().unwrap();
1314            let got = d.read_closed_enum_by_name::<Color>().ok();
1315            assert_eq!(got, want, "input: {input}");
1316        }
1317    }
1318
1319    #[test]
1320    fn read_closed_enum_unknown_number_error_kind() {
1321        let mut d = TextDecoder::new("f: 99");
1322        d.read_field_name().unwrap();
1323        let err = d.read_closed_enum_by_name::<Color>().unwrap_err();
1324        assert_eq!(err.kind, ParseErrorKind::UnknownEnumValue);
1325    }
1326
1327    // ── repeated ────────────────────────────────────────────────────────────
1328
1329    #[test]
1330    fn repeated_both_forms() {
1331        // `items: 1 items: 2 items: [3, 4] items: 5` → [1,2,3,4,5]
1332        let mut m = TestMsg::default();
1333        let mut d = TextDecoder::new("items: 1 items: 2 items: [3, 4] items: 5");
1334        m.merge_text(&mut d).unwrap();
1335        assert_eq!(m.items, vec![1, 2, 3, 4, 5]);
1336    }
1337
1338    #[test]
1339    fn repeated_empty_list() {
1340        let mut m = TestMsg::default();
1341        let mut d = TextDecoder::new("items: []");
1342        m.merge_text(&mut d).unwrap();
1343        assert_eq!(m.items, Vec::<i64>::new());
1344    }
1345
1346    #[test]
1347    fn repeated_empty_scalar_list_requires_colon() {
1348        for input in ["items []", "i: 1 items []", "child { items [] }"] {
1349            let mut m = TestMsg::default();
1350            let err = m.merge_text(&mut TextDecoder::new(input)).unwrap_err();
1351            assert_eq!(
1352                err.kind,
1353                ParseErrorKind::UnexpectedToken {
1354                    expected: "':' before scalar value",
1355                },
1356                "input: {input:?}"
1357            );
1358            assert_eq!(err.col as usize, input.find('[').unwrap() + 1);
1359        }
1360
1361        // With an element present the same error is positioned at the
1362        // element.
1363        for input in ["items [1]", "items 1"] {
1364            let mut m = TestMsg::default();
1365            let err = m.merge_text(&mut TextDecoder::new(input)).unwrap_err();
1366            assert_eq!(
1367                err.kind,
1368                ParseErrorKind::UnexpectedToken {
1369                    expected: "':' before scalar value",
1370                },
1371                "input: {input:?}"
1372            );
1373            assert_eq!(err.col as usize, input.find('1').unwrap() + 1);
1374        }
1375
1376        // The colon that counts is the list field's own. One on the
1377        // enclosing field does not stand in for it, and its absence on the
1378        // enclosing field does not matter.
1379        let mut m = TestMsg::default();
1380        let err = m
1381            .merge_text(&mut TextDecoder::new("child: { items [] }"))
1382            .unwrap_err();
1383        assert_eq!(
1384            err.kind,
1385            ParseErrorKind::UnexpectedToken {
1386                expected: "':' before scalar value",
1387            }
1388        );
1389        for input in ["child { items: [] }", "child: { items: [] } items: []"] {
1390            let mut m = TestMsg::default();
1391            m.merge_text(&mut TextDecoder::new(input)).unwrap();
1392            assert_eq!(m.items, Vec::<i64>::new(), "input: {input:?}");
1393        }
1394
1395        // An empty scalar list with its colon, inside a message list that
1396        // was opened without one.
1397        let mut d = TextDecoder::new("f [{ items: [] }, { items [] }]");
1398        d.read_field_name().unwrap();
1399        let mut out: Vec<TestMsg> = Vec::new();
1400        let err = d
1401            .read_repeated_into(&mut out, |d| {
1402                let mut m = TestMsg::default();
1403                d.merge_message(&mut m)?;
1404                Ok(m)
1405            })
1406            .unwrap_err();
1407        assert_eq!(out.len(), 1, "the first element parses");
1408        assert_eq!(
1409            err.kind,
1410            ParseErrorKind::UnexpectedToken {
1411                expected: "':' before scalar value",
1412            }
1413        );
1414    }
1415
1416    #[test]
1417    fn repeated_message_list() {
1418        // `[{...}, {...}]` — exercises ListOpen → MessageOpen in the tokenizer.
1419        // TestMsg uses `child: Option<_>` not `Vec<_>`, so use a Vec<TestMsg>
1420        // directly through read_repeated_into.
1421        let mut d = TextDecoder::new("f: [{i: 1}, {i: 2}, <i: 3>]");
1422        d.read_field_name().unwrap();
1423        let mut out: Vec<TestMsg> = Vec::new();
1424        d.read_repeated_into(&mut out, |d| {
1425            let mut m = TestMsg::default();
1426            d.merge_message(&mut m)?;
1427            Ok(m)
1428        })
1429        .unwrap();
1430        assert_eq!(out.len(), 3);
1431        assert_eq!(out[0].i, 1);
1432        assert_eq!(out[1].i, 2);
1433        assert_eq!(out[2].i, 3);
1434    }
1435
1436    #[test]
1437    fn merge_from_str_appends() {
1438        use super::super::merge_from_str;
1439        let mut m = TestMsg {
1440            i: 1,
1441            items: vec![10],
1442            ..Default::default()
1443        };
1444        merge_from_str(&mut m, "i: 2 items: 20").unwrap();
1445        assert_eq!(m.i, 2); // scalar overwritten
1446        assert_eq!(m.items, vec![10, 20]); // repeated appended
1447    }
1448
1449    // ── roundtrip ───────────────────────────────────────────────────────────
1450
1451    #[test]
1452    fn roundtrip_simple() {
1453        let orig = TestMsg {
1454            i: 42,
1455            s: String::from("hello"),
1456            items: vec![1, 2, 3],
1457            child: None,
1458        };
1459        let text = encode_to_string(&orig);
1460        assert_eq!(text, r#"i: 42 s: "hello" items: 1 items: 2 items: 3"#);
1461        let back: TestMsg = decode_from_str(&text).unwrap();
1462        assert_eq!(back, orig);
1463    }
1464
1465    #[test]
1466    fn roundtrip_nested() {
1467        let orig = TestMsg {
1468            i: 1,
1469            s: String::new(),
1470            items: vec![],
1471            child: Some(Box::new(TestMsg {
1472                i: 2,
1473                s: String::from("inner"),
1474                items: vec![10, 20],
1475                child: None,
1476            })),
1477        };
1478        let text = encode_to_string(&orig);
1479        let back: TestMsg = decode_from_str(&text).unwrap();
1480        assert_eq!(back, orig);
1481    }
1482
1483    #[test]
1484    fn roundtrip_pretty() {
1485        let orig = TestMsg {
1486            i: 1,
1487            s: String::new(),
1488            items: vec![],
1489            child: Some(Box::new(TestMsg {
1490                i: 2,
1491                s: String::new(),
1492                items: vec![],
1493                child: None,
1494            })),
1495        };
1496        let text = encode_to_string_pretty(&orig);
1497        assert_eq!(text, "i: 1\nchild {\n  i: 2\n}\n");
1498        let back: TestMsg = decode_from_str(&text).unwrap();
1499        assert_eq!(back, orig);
1500    }
1501
1502    #[test]
1503    fn parse_angle_delimiters() {
1504        // `<` `>` in place of `{` `}`.
1505        let m: TestMsg = decode_from_str("i: 1 child < i: 2 >").unwrap();
1506        assert_eq!(m.i, 1);
1507        assert_eq!(m.child.unwrap().i, 2);
1508    }
1509
1510    #[test]
1511    fn parse_canonical_cpp_output() {
1512        // What protoc --encode would produce for this message shape.
1513        let input = "i: 42\ns: \"hello\"\nitems: 1\nitems: 2\nchild {\n  i: 7\n}\n";
1514        let m: TestMsg = decode_from_str(input).unwrap();
1515        assert_eq!(m.i, 42);
1516        assert_eq!(m.s, "hello");
1517        assert_eq!(m.items, vec![1, 2]);
1518        assert_eq!(m.child.unwrap().i, 7);
1519    }
1520
1521    #[test]
1522    fn parse_with_comments_and_separators() {
1523        let input = "# header\ni: 1, # inline\ns: \"x\"; # trailing\n";
1524        let m: TestMsg = decode_from_str(input).unwrap();
1525        assert_eq!(m.i, 1);
1526        assert_eq!(m.s, "x");
1527    }
1528
1529    // ── skip_value ──────────────────────────────────────────────────────────
1530
1531    #[test]
1532    fn skip_unknown_scalar() {
1533        let m: TestMsg = decode_from_str("unknown: 999 i: 42").unwrap();
1534        assert_eq!(m.i, 42);
1535    }
1536
1537    #[test]
1538    fn skip_unknown_message() {
1539        let m: TestMsg = decode_from_str("unknown { x: 1 y: 2 } i: 42").unwrap();
1540        assert_eq!(m.i, 42);
1541    }
1542
1543    #[test]
1544    fn skip_unknown_nested_message() {
1545        let m: TestMsg = decode_from_str("unknown { inner { deep: 1 } } i: 42").unwrap();
1546        assert_eq!(m.i, 42);
1547    }
1548
1549    #[test]
1550    fn skip_unknown_list() {
1551        let m: TestMsg = decode_from_str("unknown: [1, 2, 3] i: 42").unwrap();
1552        assert_eq!(m.i, 42);
1553    }
1554
1555    #[test]
1556    fn skip_unknown_message_list() {
1557        let m: TestMsg = decode_from_str("unknown: [{a: 1}, {a: 2}] i: 42").unwrap();
1558        assert_eq!(m.i, 42);
1559    }
1560
1561    #[test]
1562    fn skip_unknown_list_without_colon() {
1563        let m: TestMsg = decode_from_str("unknown [{a: 1}, {a: 2}] i: 42").unwrap();
1564        assert_eq!(m.i, 42);
1565        let err = decode_from_str::<TestMsg>("unknown [1, 2] i: 42").unwrap_err();
1566        assert_eq!(
1567            err.kind,
1568            ParseErrorKind::UnexpectedToken {
1569                expected: "':' before scalar value",
1570            }
1571        );
1572    }
1573
1574    // ── errors ──────────────────────────────────────────────────────────────
1575
1576    #[test]
1577    fn unknown_field_error() {
1578        // A merge_text impl that doesn't skip unknowns.
1579        #[derive(Default, Clone, PartialEq, Debug)]
1580        struct Strict {
1581            i: i32,
1582        }
1583        impl crate::DefaultInstance for Strict {
1584            fn default_instance() -> &'static Self {
1585                static I: crate::__private::OnceBox<Strict> = crate::__private::OnceBox::new();
1586                I.get_or_init(|| Box::new(Strict::default()))
1587            }
1588        }
1589        impl crate::Message for Strict {
1590            fn compute_size(&self, _: &mut crate::SizeCache) -> u32 {
1591                0
1592            }
1593            fn write_to(&self, _: &mut crate::SizeCache, _: &mut impl crate::EncodeSink) {}
1594            fn merge_field(
1595                &mut self,
1596                t: crate::encoding::Tag,
1597                b: &mut impl bytes::Buf,
1598                _: crate::DecodeContext<'_>,
1599            ) -> Result<(), crate::DecodeError> {
1600                crate::encoding::skip_field(t, b)
1601            }
1602            fn clear(&mut self) {}
1603        }
1604        impl TextFormat for Strict {
1605            fn encode_text(&self, _: &mut TextEncoder<'_>) -> core::fmt::Result {
1606                Ok(())
1607            }
1608            fn merge_text(&mut self, d: &mut TextDecoder<'_>) -> Result<(), ParseError> {
1609                while let Some(name) = d.read_field_name()? {
1610                    match name {
1611                        "i" => self.i = d.read_i32()?,
1612                        _ => return Err(d.unknown_field()),
1613                    }
1614                }
1615                Ok(())
1616            }
1617        }
1618
1619        let err = decode_from_str::<Strict>("i: 1\nbad: 2").unwrap_err();
1620        assert_eq!(err.kind, ParseErrorKind::UnknownField);
1621        assert_eq!(err.line, 2);
1622        assert_eq!(err.col, 1);
1623    }
1624
1625    #[test]
1626    fn depth_limit_exceeded() {
1627        // Build a chain of `child { child { ... } }` past the limit.
1628        let depth = crate::message::RECURSION_LIMIT as usize + 1;
1629        let mut s = String::new();
1630        for _ in 0..depth {
1631            s.push_str("child { ");
1632        }
1633        for _ in 0..depth {
1634            s.push('}');
1635        }
1636        let err = decode_from_str::<TestMsg>(&s).unwrap_err();
1637        assert_eq!(err.kind, ParseErrorKind::RecursionLimitExceeded);
1638    }
1639
1640    #[test]
1641    fn error_position_in_value() {
1642        // Error on line 2, in the value.
1643        let err = decode_from_str::<TestMsg>("i: 1\ni: notanumber").unwrap_err();
1644        assert_eq!(err.line, 2);
1645        assert!(matches!(err.kind, ParseErrorKind::UnexpectedToken { .. }));
1646    }
1647}