buffa/text/decoder.rs
1// Modified by ReallyMe Technologies Limited.
2//! Textproto decoder.
3//!
4//! Wraps a [`Tokenizer`] and adds type-directed value interpretation. A
5//! generated `merge_text` implementation calls
6//! [`read_field_name`](TextDecoder::read_field_name) in a loop, dispatches on
7//! the returned name, and calls the appropriate `read_*` method for that
8//! field's type.
9
10use alloc::borrow::Cow;
11use alloc::vec::Vec;
12
13use super::error::{ParseError, ParseErrorKind};
14use super::string::{unescape, unescape_str, UnescapeError};
15use super::token::{
16 consume_ws, lex_number, normalize_bracket_name, number_for_parse, NumKind, ScalarKind, Token,
17 TokenKind, Tokenizer,
18};
19
20/// Stateful textproto reader.
21///
22/// Drives a [`Tokenizer`] and interprets scalar tokens as the requested Rust
23/// type. Recursion depth is enforced by the tokenizer's fixed-size open-stack
24/// (see [`RECURSION_LIMIT`](crate::RECURSION_LIMIT)), and repeated and map
25/// elements are charged against an element-memory budget defaulting to
26/// [`DEFAULT_ELEMENT_MEMORY_LIMIT`](crate::DEFAULT_ELEMENT_MEMORY_LIMIT)
27/// (see [`with_element_memory_limit`](Self::with_element_memory_limit)).
28pub struct TextDecoder<'a> {
29 tok: Tokenizer<'a>,
30 /// Byte position of the last name returned by `read_field_name`, for
31 /// [`unknown_field`](Self::unknown_field) error reporting.
32 last_name_pos: usize,
33 /// Remaining element-memory budget, shared across the whole parse.
34 ///
35 /// The binary decoder's equivalent rides on `DecodeContext`, which never
36 /// reaches this parser, so textproto carries its own. See
37 /// [`ParseErrorKind::ElementMemoryLimitExceeded`] for the amplification
38 /// it bounds.
39 elem_remaining: usize,
40}
41
42impl<'a> TextDecoder<'a> {
43 /// Create a decoder over `input`.
44 pub fn new(input: &'a str) -> Self {
45 Self {
46 tok: Tokenizer::new(input),
47 last_name_pos: 0,
48 elem_remaining: crate::DEFAULT_ELEMENT_MEMORY_LIMIT,
49 }
50 }
51
52 /// Set the element-memory budget for this parse.
53 ///
54 /// Bounds what the parse *materializes* rather than what it reads; see
55 /// [`ParseErrorKind::ElementMemoryLimitExceeded`]. Defaults to
56 /// [`DEFAULT_ELEMENT_MEMORY_LIMIT`](crate::DEFAULT_ELEMENT_MEMORY_LIMIT);
57 /// pass `usize::MAX` to disable.
58 ///
59 /// The budget is shared across the whole parse, not per field, as
60 /// [`DecodeOptions::with_element_memory_limit`](crate::DecodeOptions::with_element_memory_limit)
61 /// is on the binary side. The two are not charged identically: this
62 /// parser also charges repeated scalars, which the binary decoder
63 /// exempts because a packed 1-byte varint becoming a 4-byte `i32` is not
64 /// an amplification vector. Text has no packed form, so a repeated
65 /// scalar costs at least two input bytes per element and cannot reach a
66 /// worse ratio than that exemption protects — but a textproto scalar
67 /// list above roughly `limit / size_of::<T>()` elements is refused where
68 /// the equivalent binary payload is not.
69 #[must_use]
70 pub fn with_element_memory_limit(mut self, bytes: usize) -> Self {
71 self.elem_remaining = bytes;
72 self
73 }
74
75 /// Charge one element of type `T` against the budget.
76 ///
77 /// Charged before the element is parsed, so an over-budget input costs
78 /// the check rather than the work.
79 ///
80 /// Every repeated element is charged, including scalars — which the
81 /// binary decoder deliberately exempts, on the grounds that a packed
82 /// 1-byte varint becoming a 4-byte `i32` is not an amplification vector
83 /// and bounding it would reject columnar payloads. Text has no packed
84 /// form, so a repeated scalar costs at least two input bytes per element
85 /// and cannot reach a worse ratio than that exemption protects; charging
86 /// it uniformly keeps this one function the whole story rather than
87 /// pushing an element-kind distinction through codegen. The practical
88 /// effect is that a textproto scalar list above roughly
89 /// `DEFAULT_ELEMENT_MEMORY_LIMIT / size_of::<T>()` elements is refused
90 /// where the equivalent binary payload is not.
91 fn charge_element<T>(&mut self) -> Result<(), ParseError> {
92 let cost = core::mem::size_of::<T>();
93 if cost > self.elem_remaining {
94 return Err(self
95 .tok
96 .err_here(ParseErrorKind::ElementMemoryLimitExceeded));
97 }
98 self.elem_remaining -= cost;
99 Ok(())
100 }
101
102 /// Construct a parse error at a token's position.
103 #[inline]
104 fn err_at(&self, tok: &Token<'_>, kind: ParseErrorKind) -> ParseError {
105 let (line, col) = self.tok.line_col(tok.pos);
106 ParseError::new(line, col, kind)
107 }
108
109 /// Read the next field name, or return `None` at end-of-message / EOF.
110 ///
111 /// The returned slice is borrowed directly from the input. For bracketed
112 /// type names (`[pkg.ext]`) it includes the brackets — generated code
113 /// matches against `"[pkg.ext]"` literally.
114 ///
115 /// The `:` after the name is optional before a message value or a list
116 /// of message values. This method returns the name either way; for a
117 /// scalar value, or a scalar list element, that follows a name without
118 /// one, the call that reads the value returns the error. An empty list
119 /// `[]` has no element to read, so it parses with or without the colon,
120 /// unless the caller reads it with
121 /// [`read_repeated_scalar_into`](Self::read_repeated_scalar_into), which
122 /// requires the colon.
123 ///
124 /// # Errors
125 ///
126 /// Any tokenizer error — malformed name, delimiter mismatch, etc.
127 pub fn read_field_name(&mut self) -> Result<Option<&'a str>, ParseError> {
128 let tok = self.tok.peek()?;
129 match tok.kind {
130 TokenKind::Eof | TokenKind::MessageClose => Ok(None),
131 TokenKind::Name => {
132 self.tok.read()?;
133 self.last_name_pos = tok.pos;
134 Ok(Some(tok.raw))
135 }
136 _ => Err(self.err_at(
137 &tok,
138 ParseErrorKind::UnexpectedToken {
139 expected: "field name",
140 },
141 )),
142 }
143 }
144
145 /// Construct an unknown-field error pointing at the last name returned
146 /// by [`read_field_name`](Self::read_field_name).
147 ///
148 /// Generated `merge_text` wildcard arms return this for any name that is
149 /// neither a declared field nor a declared reserved name; reserved names
150 /// are consumed with [`skip_value`](Self::skip_value) instead.
151 pub fn unknown_field(&self) -> ParseError {
152 let (line, col) = self.tok.line_col(self.last_name_pos);
153 ParseError::new(line, col, ParseErrorKind::UnknownField)
154 }
155
156 /// Read a scalar token, checking it has the right kind.
157 fn read_scalar(&mut self, want: ScalarKind) -> Result<Token<'a>, ParseError> {
158 let tok = self.tok.read()?;
159 if tok.kind != TokenKind::Scalar || tok.scalar_kind != want {
160 return Err(self.err_at(
161 &tok,
162 ParseErrorKind::UnexpectedToken {
163 expected: match want {
164 ScalarKind::Number => "number",
165 ScalarKind::String => "string literal",
166 ScalarKind::Literal => "identifier",
167 },
168 },
169 ));
170 }
171 Ok(tok)
172 }
173
174 /// Parse a number token's text as an integer. Shared by all integer readers.
175 ///
176 /// Strips hex/oct prefixes and dispatches to the appropriate radix parser.
177 /// Rejects floats and (for unsigned) negatives.
178 fn parse_int<T>(
179 &self,
180 tok: &Token<'_>,
181 signed: bool,
182 from_dec: fn(&str) -> Option<T>,
183 from_radix: fn(&str, u32) -> Option<T>,
184 ) -> Result<T, ParseError> {
185 let num = lex_number(tok.raw.as_bytes())
186 .ok_or_else(|| self.err_at(tok, ParseErrorKind::InvalidNumber))?;
187 if num.kind == NumKind::Float {
188 return Err(self.err_at(tok, ParseErrorKind::InvalidNumber));
189 }
190 if !signed && num.neg {
191 return Err(self.err_at(tok, ParseErrorKind::InvalidNumber));
192 }
193 let cow = number_for_parse(tok.raw, &num);
194 let s: &str = &cow;
195 let parsed = match num.kind {
196 NumKind::Dec => from_dec(s),
197 NumKind::Hex | NumKind::Oct => {
198 // Strip `-` (if any), strip the base prefix, then parse with
199 // the radix. For signed negative hex/oct, re-prepend `-` —
200 // `i32::from_str_radix("-1F", 16)` works.
201 let (neg, rest) = match s.strip_prefix('-') {
202 Some(r) => (true, r),
203 None => (false, s),
204 };
205 let (radix, digits) = if num.kind == NumKind::Hex {
206 let d = rest
207 .strip_prefix("0x")
208 .or_else(|| rest.strip_prefix("0X"))
209 .ok_or_else(|| self.err_at(tok, ParseErrorKind::InvalidNumber))?;
210 (16, d)
211 } else {
212 // Oct: lex only flags Oct when there IS a leading 0 and
213 // at least one more digit.
214 (8, rest.strip_prefix('0').unwrap_or(rest))
215 };
216 if !neg {
217 from_radix(digits, radix)
218 } else {
219 // Stitch `-` + digits for signed radix parse.
220 let mut tmp = alloc::string::String::with_capacity(1 + digits.len());
221 tmp.push('-');
222 tmp.push_str(digits);
223 from_radix(&tmp, radix)
224 }
225 }
226 NumKind::Float => unreachable!("rejected above"),
227 };
228 parsed.ok_or_else(|| self.err_at(tok, ParseErrorKind::InvalidNumber))
229 }
230
231 /// Read an `i32` value.
232 ///
233 /// Accepts decimal, `0x` hex, `0` octal. Rejects floats and out-of-range.
234 ///
235 /// # Errors
236 ///
237 /// [`ParseErrorKind::InvalidNumber`] if not a valid in-range integer, or
238 /// [`ParseErrorKind::UnexpectedToken`] if the value is not a number at all.
239 pub fn read_i32(&mut self) -> Result<i32, ParseError> {
240 let tok = self.read_scalar(ScalarKind::Number)?;
241 self.parse_int(
242 &tok,
243 true,
244 |s| s.parse().ok(),
245 |s, r| i32::from_str_radix(s, r).ok(),
246 )
247 }
248
249 /// Read an `i64` value.
250 ///
251 /// # Errors
252 ///
253 /// As [`read_i32`](Self::read_i32).
254 pub fn read_i64(&mut self) -> Result<i64, ParseError> {
255 let tok = self.read_scalar(ScalarKind::Number)?;
256 self.parse_int(
257 &tok,
258 true,
259 |s| s.parse().ok(),
260 |s, r| i64::from_str_radix(s, r).ok(),
261 )
262 }
263
264 /// Read a `u32` value. Rejects negatives.
265 ///
266 /// # Errors
267 ///
268 /// As [`read_i32`](Self::read_i32).
269 pub fn read_u32(&mut self) -> Result<u32, ParseError> {
270 let tok = self.read_scalar(ScalarKind::Number)?;
271 self.parse_int(
272 &tok,
273 false,
274 |s| s.parse().ok(),
275 |s, r| u32::from_str_radix(s, r).ok(),
276 )
277 }
278
279 /// Read a `u64` value. Rejects negatives.
280 ///
281 /// # Errors
282 ///
283 /// As [`read_i32`](Self::read_i32).
284 pub fn read_u64(&mut self) -> Result<u64, ParseError> {
285 let tok = self.read_scalar(ScalarKind::Number)?;
286 self.parse_int(
287 &tok,
288 false,
289 |s| s.parse().ok(),
290 |s, r| u64::from_str_radix(s, r).ok(),
291 )
292 }
293
294 /// Read an `f32` value.
295 ///
296 /// Accepts a decimal number, optionally with an `f` suffix, or one of the
297 /// case-insensitive literals `nan`, `inf` and `infinity`. Either can
298 /// follow a `-`, and whitespace or comments can separate the sign from
299 /// what it negates.
300 ///
301 /// A number rounds once, to the nearest value of the type, ties to even.
302 /// Overflow gives ±∞ and underflow gives ±0.
303 ///
304 /// # Errors
305 ///
306 /// [`ParseErrorKind::InvalidNumber`] if the token is a hex or octal
307 /// number, or an identifier other than `nan`, `inf` or `infinity`.
308 /// [`ParseErrorKind::UnexpectedToken`] if the token is a string or is not
309 /// a scalar.
310 pub fn read_f32(&mut self) -> Result<f32, ParseError> {
311 self.read_float(f32::NAN, f32::INFINITY)
312 }
313
314 /// Read an `f64` value. See [`read_f32`](Self::read_f32).
315 ///
316 /// # Errors
317 ///
318 /// As [`read_f32`](Self::read_f32).
319 pub fn read_f64(&mut self) -> Result<f64, ParseError> {
320 self.read_float(f64::NAN, f64::INFINITY)
321 }
322
323 fn read_float<T>(&mut self, nan: T, infinity: T) -> Result<T, ParseError>
324 where
325 T: core::str::FromStr + core::ops::Neg<Output = T> + Copy,
326 {
327 let tok = self.tok.read()?;
328 if tok.kind != TokenKind::Scalar {
329 return Err(self.err_at(&tok, ParseErrorKind::UnexpectedToken { expected: "number" }));
330 }
331 match tok.scalar_kind {
332 ScalarKind::Literal => {
333 // nan, inf, infinity, -inf, -infinity (case-insensitive).
334 // The tokenizer accepts whitespace/comments between `-` and
335 // the literal, so reuse its trivia handling for the raw span.
336 let (neg, lit) = match tok.raw.strip_prefix('-') {
337 Some(r) => {
338 // `consume_ws` only stops after an ASCII whitespace
339 // byte or a comment's `\n`, neither of which can sit
340 // inside a multi-byte sequence, so the offset is a
341 // char boundary even when a comment holds non-ASCII.
342 let trivia_len = r.len() - consume_ws(r.as_bytes()).len();
343 debug_assert!(r.is_char_boundary(trivia_len));
344 (true, &r[trivia_len..])
345 }
346 None => (false, tok.raw),
347 };
348 let value = if lit.eq_ignore_ascii_case("nan") {
349 nan
350 } else if lit.eq_ignore_ascii_case("inf") || lit.eq_ignore_ascii_case("infinity") {
351 infinity
352 } else {
353 return Err(self.err_at(&tok, ParseErrorKind::InvalidNumber));
354 };
355 Ok(if neg { -value } else { value })
356 }
357 ScalarKind::Number => {
358 let num = lex_number(tok.raw.as_bytes())
359 .ok_or_else(|| self.err_at(&tok, ParseErrorKind::InvalidNumber))?;
360 match num.kind {
361 NumKind::Dec | NumKind::Float => {
362 // Rust's float parse saturates to ±∞ on overflow and
363 // to ±0.0 on underflow (since 1.55), both of which are
364 // the behaviours the textproto spec requires. The sign
365 // is preserved: `"-0"` and `"-1e-400"` parse to -0.0.
366 //
367 // The parse goes straight to `T` so that it rounds
368 // once. Reading an `f32` as an `f64` and narrowing
369 // rounds twice, and for a decimal just off the
370 // midpoint of two `f32` values the result is one ulp
371 // from the nearest `f32`.
372 number_for_parse(tok.raw, &num)
373 .parse::<T>()
374 .map_err(|_| self.err_at(&tok, ParseErrorKind::InvalidNumber))
375 }
376 NumKind::Hex | NumKind::Oct => {
377 // The textproto spec's FLOAT production is base-10
378 // only; `0x1` and `01` are not float literals. The
379 // conformance suite explicitly tests rejection.
380 Err(self.err_at(&tok, ParseErrorKind::InvalidNumber))
381 }
382 }
383 }
384 ScalarKind::String => {
385 Err(self.err_at(&tok, ParseErrorKind::UnexpectedToken { expected: "number" }))
386 }
387 }
388 }
389
390 /// Read a `bool` value.
391 ///
392 /// Accepts `true`, `True`, `t`, `false`, `False`, `f`, and `0`/`1`
393 /// (in any integer base). These are the exact literals the C++ text
394 /// parser accepts.
395 ///
396 /// # Errors
397 ///
398 /// [`ParseErrorKind::UnexpectedToken`] if the token is not a recognised
399 /// boolean form.
400 pub fn read_bool(&mut self) -> Result<bool, ParseError> {
401 let tok = self.tok.read()?;
402 if tok.kind != TokenKind::Scalar {
403 return Err(self.err_at(
404 &tok,
405 ParseErrorKind::UnexpectedToken {
406 expected: "boolean",
407 },
408 ));
409 }
410 match tok.scalar_kind {
411 ScalarKind::Literal => match tok.raw {
412 "true" | "True" | "t" => Ok(true),
413 "false" | "False" | "f" => Ok(false),
414 _ => Err(self.err_at(
415 &tok,
416 ParseErrorKind::UnexpectedToken {
417 expected: "boolean",
418 },
419 )),
420 },
421 ScalarKind::Number => {
422 // 0/1 in any base (00, 0x1, 01 all accepted by C++).
423 let n: u64 = self.parse_int(
424 &tok,
425 false,
426 |s| s.parse().ok(),
427 |s, r| u64::from_str_radix(s, r).ok(),
428 )?;
429 match n {
430 0 => Ok(false),
431 1 => Ok(true),
432 _ => Err(self.err_at(
433 &tok,
434 ParseErrorKind::UnexpectedToken {
435 expected: "boolean",
436 },
437 )),
438 }
439 }
440 ScalarKind::String => Err(self.err_at(
441 &tok,
442 ParseErrorKind::UnexpectedToken {
443 expected: "boolean",
444 },
445 )),
446 }
447 }
448
449 /// Read a `string` value. Unescapes and UTF-8-validates.
450 ///
451 /// Borrows the input when the token is a single literal with no escapes.
452 ///
453 /// # Errors
454 ///
455 /// [`ParseErrorKind::InvalidString`] for malformed escapes,
456 /// [`ParseErrorKind::InvalidUtf8`] if the unescaped bytes are not valid
457 /// UTF-8.
458 pub fn read_string(&mut self) -> Result<Cow<'a, str>, ParseError> {
459 let tok = self.read_scalar(ScalarKind::String)?;
460 unescape_str(tok.raw).map_err(|e| {
461 self.err_at(
462 &tok,
463 match e {
464 UnescapeError::InvalidUtf8 => ParseErrorKind::InvalidUtf8,
465 UnescapeError::BadEscape(why) => ParseErrorKind::InvalidString(why),
466 },
467 )
468 })
469 }
470
471 /// Read a `bytes` value. Unescapes but does not UTF-8-validate.
472 ///
473 /// # Errors
474 ///
475 /// [`ParseErrorKind::InvalidString`] for malformed escapes.
476 pub fn read_bytes(&mut self) -> Result<Vec<u8>, ParseError> {
477 let tok = self.read_scalar(ScalarKind::String)?;
478 unescape(tok.raw).map_err(|e| {
479 // unescape (byte-level) never returns InvalidUtf8.
480 let UnescapeError::BadEscape(why) = e else {
481 unreachable!("unescape is byte-level")
482 };
483 self.err_at(&tok, ParseErrorKind::InvalidString(why))
484 })
485 }
486
487 /// Read an enum token into `(i32_value, had_known_name, token)`.
488 ///
489 /// Shared by the open and closed enum read methods. For a named variant,
490 /// looks up via [`Enumeration::from_proto_name`](crate::Enumeration::from_proto_name)
491 /// and `had_known_name` is true. For a numeric form, parses the i32 and
492 /// `had_known_name` is false. The caller decides whether unknown numbers
493 /// are acceptable.
494 fn read_enum_inner<E: crate::Enumeration>(
495 &mut self,
496 ) -> Result<(i32, bool, Token<'a>), ParseError> {
497 let tok = self.tok.read()?;
498 if tok.kind != TokenKind::Scalar {
499 return Err(self.err_at(
500 &tok,
501 ParseErrorKind::UnexpectedToken {
502 expected: "enum value",
503 },
504 ));
505 }
506 match tok.scalar_kind {
507 ScalarKind::Literal if !tok.raw.starts_with('-') => E::from_proto_name(tok.raw)
508 .map(|e| (e.to_i32(), true, tok))
509 .ok_or_else(|| self.err_at(&tok, ParseErrorKind::UnknownEnumValue)),
510 ScalarKind::Number => {
511 let n = self.parse_int(
512 &tok,
513 true,
514 |s| s.parse().ok(),
515 |s, r| i32::from_str_radix(s, r).ok(),
516 )?;
517 Ok((n, false, tok))
518 }
519 _ => Err(self.err_at(
520 &tok,
521 ParseErrorKind::UnexpectedToken {
522 expected: "enum value",
523 },
524 )),
525 }
526 }
527
528 /// Read an enum value by variant name or by number (open-enum semantics).
529 ///
530 /// Returns the `i32` wire value. Any in-range integer is accepted — the
531 /// proto3 open-enum model preserves unknown numeric values.
532 ///
533 /// # Errors
534 ///
535 /// [`ParseErrorKind::UnknownEnumValue`] if the name is not a known
536 /// variant; [`ParseErrorKind::InvalidNumber`] if a numeric form is out
537 /// of i32 range.
538 pub fn read_enum_by_name<E: crate::Enumeration>(&mut self) -> Result<i32, ParseError> {
539 self.read_enum_inner::<E>().map(|(n, _, _)| n)
540 }
541
542 /// Read a closed-enum value by variant name or by number.
543 ///
544 /// Returns the enum variant directly. Unknown numeric values are
545 /// **rejected** — the proto2 closed-enum model does not accept values
546 /// outside the defined set in text format. (Binary decode routes them to
547 /// unknown fields; text format has no analogous mechanism, so it errors.)
548 ///
549 /// # Errors
550 ///
551 /// [`ParseErrorKind::UnknownEnumValue`] if the name is not a known
552 /// variant, **or** if a numeric form does not map to a defined variant.
553 pub fn read_closed_enum_by_name<E: crate::Enumeration>(&mut self) -> Result<E, ParseError> {
554 let (n, by_name, tok) = self.read_enum_inner::<E>()?;
555 if by_name {
556 // from_proto_name already succeeded; from_i32 on its to_i32()
557 // is infallible. Avoid a second name lookup.
558 return Ok(E::from_i32(n).expect("from_proto_name returned a valid variant"));
559 }
560 E::from_i32(n).ok_or_else(|| self.err_at(&tok, ParseErrorKind::UnknownEnumValue))
561 }
562
563 /// Enter a `{` or `<`, merge into `msg`, then consume the matching close.
564 ///
565 /// # Errors
566 ///
567 /// Any tokenizer or `merge_text` error, including
568 /// [`ParseErrorKind::RecursionLimitExceeded`] if nesting exceeds
569 /// [`RECURSION_LIMIT`](crate::RECURSION_LIMIT).
570 pub fn merge_message<M: super::TextFormat>(&mut self, msg: &mut M) -> Result<(), ParseError> {
571 self.merge_map_entry(|dec| msg.merge_text(dec))
572 }
573
574 /// Consume `{` or `<`, run `f` over the body, then consume the close.
575 ///
576 /// Closure-taking counterpart to [`merge_message`](Self::merge_message) —
577 /// same rationale as [`TextEncoder::write_map_entry`]. Generated
578 /// `map<K, V>` decode captures `&mut Option<K>` / `&mut Option<V>` and
579 /// dispatches on `"key"` / `"value"` inside the closure without naming a
580 /// concrete entry type.
581 ///
582 /// `#[doc(hidden)]` — codegen support, not public API.
583 ///
584 /// # Errors
585 ///
586 /// Any tokenizer or `f` error.
587 #[doc(hidden)]
588 pub fn merge_map_entry(
589 &mut self,
590 f: impl FnOnce(&mut Self) -> Result<(), ParseError>,
591 ) -> Result<(), ParseError> {
592 let open = self.tok.read()?;
593 if open.kind != TokenKind::MessageOpen {
594 return Err(self.err_at(
595 &open,
596 ParseErrorKind::UnexpectedToken {
597 expected: "'{' or '<'",
598 },
599 ));
600 }
601 f(self)?;
602 let close = self.tok.read()?;
603 if close.kind != TokenKind::MessageClose {
604 return Err(self.err_at(
605 &close,
606 ParseErrorKind::UnexpectedToken {
607 expected: "'}' or '>'",
608 },
609 ));
610 }
611 Ok(())
612 }
613
614 /// Read one or more values of a repeated field into `out`.
615 ///
616 /// Handles both forms of a repeated value: the list `f: [1, 2, 3]`
617 /// (consumes `[` and `]`) and the single value `f: 1` (reads exactly one
618 /// element). Generated code calls this once per `f` occurrence; the
619 /// `f: 1 f: 2` form is handled by the outer `read_field_name` loop seeing
620 /// `f` twice.
621 ///
622 /// An empty list `[]` is accepted with or without a `:` after the field
623 /// name, which is the rule for a list of messages. For a field whose
624 /// elements are scalars, use
625 /// [`read_repeated_scalar_into`](Self::read_repeated_scalar_into), which
626 /// requires the colon.
627 ///
628 /// # Errors
629 ///
630 /// Any error from `read_one` or the tokenizer.
631 pub fn read_repeated_into<T>(
632 &mut self,
633 out: &mut Vec<T>,
634 read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
635 ) -> Result<(), ParseError> {
636 self.read_repeated_inner(|value| out.push(value), read_one, false)
637 }
638
639 /// Read one or more values of a repeated scalar field into `out`,
640 /// requiring the `:` after the field name even before an empty list.
641 ///
642 /// Reads the same two forms as
643 /// [`read_repeated_into`](Self::read_repeated_into). Use it when the
644 /// field's elements are numbers, bools, enums, strings or bytes, as
645 /// generated code does. The tokenizer cannot infer the field type from
646 /// `[]`, so this method applies the check for the empty list. For a
647 /// list with elements, or a single value, the tokenizer finds the
648 /// missing colon when this method reads the element.
649 ///
650 /// # Errors
651 ///
652 /// - [`ParseErrorKind::UnexpectedToken`] with `expected` set to
653 /// `"':' before scalar value"` when the field name has no `:` after
654 /// it. The error is positioned at the `[` of an empty list, and at the
655 /// element for a list with elements or a single value.
656 /// - Any other error from `read_one` or the tokenizer.
657 pub fn read_repeated_scalar_into<T>(
658 &mut self,
659 out: &mut Vec<T>,
660 read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
661 ) -> Result<(), ParseError> {
662 self.read_repeated_inner(|value| out.push(value), read_one, true)
663 }
664
665 /// Decode repeated messages directly into a custom collection.
666 /// # Errors
667 /// Returns a tokenizer or element-decoding error, preserving earlier elements.
668 pub fn read_repeated_into_storage<T, C: crate::ProtoList<T>>(
669 &mut self,
670 out: &mut C,
671 read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
672 ) -> Result<(), ParseError> {
673 self.read_repeated_inner(|value| crate::ProtoList::push(out, value), read_one, false)
674 }
675 /// Decode repeated scalars directly into a custom collection, requiring `:`.
676 /// # Errors
677 /// Returns a tokenizer or element-decoding error, preserving earlier elements.
678 pub fn read_repeated_scalar_into_storage<T, C: crate::ProtoList<T>>(
679 &mut self,
680 out: &mut C,
681 read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
682 ) -> Result<(), ParseError> {
683 self.read_repeated_inner(|value| crate::ProtoList::push(out, value), read_one, true)
684 }
685
686 /// Decode repeated message values and pass each one directly to its owner.
687 /// # Errors
688 /// Returns a tokenizer or element-decoding error after committing earlier values.
689 pub fn read_repeated_with<T>(
690 &mut self,
691 push: impl FnMut(T),
692 read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
693 ) -> Result<(), ParseError> {
694 self.read_repeated_inner(push, read_one, false)
695 }
696
697 fn read_repeated_inner<T>(
698 &mut self,
699 mut push: impl FnMut(T),
700 mut read_one: impl FnMut(&mut Self) -> Result<T, ParseError>,
701 empty_list_requires_separator: bool,
702 ) -> Result<(), ParseError> {
703 if self.tok.peek()?.kind == TokenKind::ListOpen {
704 let open = self.tok.read()?;
705 if self.tok.peek()?.kind == TokenKind::ListClose {
706 if empty_list_requires_separator && self.tok.in_list_without_separator() {
707 return Err(self.err_at(
708 &open,
709 ParseErrorKind::UnexpectedToken {
710 expected: super::token::COLON_BEFORE_SCALAR,
711 },
712 ));
713 }
714 self.tok.read()?; // empty: `[]`
715 return Ok(());
716 }
717 loop {
718 self.charge_element::<T>()?;
719 push(read_one(self)?);
720 // The tokenizer consumes the comma between elements. When
721 // there are no more elements, `ListClose` is next.
722 if self.tok.peek()?.kind == TokenKind::ListClose {
723 self.tok.read()?;
724 return Ok(());
725 }
726 }
727 }
728 // Single-value form.
729 self.charge_element::<T>()?;
730 push(read_one(self)?);
731 Ok(())
732 }
733
734 /// Parse an `Any`-expansion body: the `[type_url] { fields }` form.
735 ///
736 /// `name` is the bracketed name as returned by
737 /// [`read_field_name`](Self::read_field_name), e.g.
738 /// `"[type.googleapis.com/pkg.Foo]"`. The brackets, and any whitespace or
739 /// `#` comments between them, are stripped here and the result is looked
740 /// up in the global text-format `Any` map (installed via
741 /// [`set_type_registry`]); the registered `text_merge` then consumes the
742 /// `{ ... }` body and re-encodes to wire bytes suitable for `Any.value`.
743 ///
744 /// Returns `(type_url, value_bytes)`, where `type_url` is the URL between
745 /// the brackets with whitespace and comments removed. It is the value for
746 /// `Any.type_url`, and is borrowed from `name` unless whitespace or
747 /// comments were removed. Its prefix can differ from the registered
748 /// URL's, because the map matches an unregistered URL by the message full
749 /// name after its last `/`.
750 ///
751 /// # Errors
752 ///
753 /// [`ParseErrorKind::UnknownField`] if neither the URL nor its message
754 /// full name is registered — this matches `AnyFieldWithInvalidType` in
755 /// the conformance suite, which expects parse failure on an unknown URL.
756 ///
757 /// [`set_type_registry`]: crate::type_registry::set_type_registry
758 pub fn read_any_expansion(
759 &mut self,
760 name: &'a str,
761 ) -> Result<(Cow<'a, str>, Vec<u8>), ParseError> {
762 // read_field_name only returns bracketed names for NameKind::TypeName;
763 // a missing bracket here means the caller dispatched wrong.
764 let url = normalize_bracket_name(name).ok_or_else(|| self.unknown_field())?;
765 let entry =
766 crate::type_registry::global_text_any(&url).ok_or_else(|| self.unknown_field())?;
767 let bytes = (entry.text_merge)(self)?;
768 Ok((url, bytes))
769 }
770
771 /// Parse an extension bracket body: the `[pkg.ext] { ... }` form.
772 ///
773 /// `name` is the bracketed name as returned by
774 /// [`read_field_name`](Self::read_field_name). The brackets, and any
775 /// whitespace or `#` comments between them, are stripped and the result
776 /// is looked up by `full_name` in the global text-format
777 /// extension map (installed via [`set_type_registry`]); the registered
778 /// `text_merge` consumes the value and produces unknown-field records at
779 /// the extension's field number.
780 ///
781 /// Repeated message and group entries accept a message list or a single
782 /// message value and produce one record per element.
783 ///
784 /// # Errors
785 ///
786 /// [`ParseErrorKind::UnknownField`] if the name is not registered or the
787 /// registered entry extends a different message. Strict by default —
788 /// protobuf-go's `prototext` behaviour, and what the
789 /// `GroupFieldExtensionGroupName` conformance test expects.
790 ///
791 /// [`set_type_registry`]: crate::type_registry::set_type_registry
792 pub fn read_extension(
793 &mut self,
794 name: &str,
795 extendee: &str,
796 ) -> Result<Vec<crate::UnknownField>, ParseError> {
797 let full = normalize_bracket_name(name).ok_or_else(|| self.unknown_field())?;
798 let Some(entry) = crate::type_registry::global_text_ext_by_name(full.as_ref()) else {
799 return Err(self.unknown_field());
800 };
801 if entry.extendee != extendee {
802 return Err(self.unknown_field());
803 }
804 (entry.text_merge)(self, entry.number)
805 }
806
807 /// Consume a field's value without interpreting it.
808 ///
809 /// Generated `merge_text` impls call this for declared reserved field
810 /// names; hand-written impls can call it for any name they want to accept
811 /// and discard rather than fail on. Handles scalars, messages (`{}` and
812 /// `<>`), and lists (recursively).
813 ///
814 /// # Errors
815 ///
816 /// Any tokenizer error in the skipped span.
817 pub fn skip_value(&mut self) -> Result<(), ParseError> {
818 let tok = self.tok.peek()?;
819 match tok.kind {
820 TokenKind::Scalar => {
821 self.tok.read()?;
822 Ok(())
823 }
824 TokenKind::MessageOpen => {
825 self.tok.read()?;
826 // read_field_name returns None at MessageClose — loop until then.
827 while self.read_field_name()?.is_some() {
828 self.skip_value()?;
829 }
830 // Consume the MessageClose.
831 let close = self.tok.read()?;
832 debug_assert_eq!(close.kind, TokenKind::MessageClose);
833 Ok(())
834 }
835 TokenKind::ListOpen => {
836 self.tok.read()?;
837 loop {
838 if self.tok.peek()?.kind == TokenKind::ListClose {
839 self.tok.read()?;
840 return Ok(());
841 }
842 self.skip_value()?;
843 }
844 }
845 _ => Err(self.err_at(&tok, ParseErrorKind::UnexpectedToken { expected: "value" })),
846 }
847 }
848}
849
850// ─────────────────────────────────────────────────────────────────────────────
851// Tests
852// ─────────────────────────────────────────────────────────────────────────────
853
854#[cfg(test)]
855mod tests {
856 use super::super::encoder::TextEncoder;
857 use super::super::{decode_from_str, encode_to_string, encode_to_string_pretty, TextFormat};
858 use super::*;
859 use alloc::boxed::Box;
860 use alloc::string::String;
861 use alloc::vec;
862
863 // ── test enum ───────────────────────────────────────────────────────────
864
865 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
866 enum Color {
867 Red,
868 Green,
869 Blue,
870 }
871
872 impl crate::Enumeration for Color {
873 fn from_i32(v: i32) -> Option<Self> {
874 match v {
875 0 => Some(Color::Red),
876 1 => Some(Color::Green),
877 2 => Some(Color::Blue),
878 _ => None,
879 }
880 }
881 fn to_i32(&self) -> i32 {
882 match self {
883 Color::Red => 0,
884 Color::Green => 1,
885 Color::Blue => 2,
886 }
887 }
888 fn proto_name(&self) -> &'static str {
889 match self {
890 Color::Red => "RED",
891 Color::Green => "GREEN",
892 Color::Blue => "BLUE",
893 }
894 }
895 fn from_proto_name(name: &str) -> Option<Self> {
896 match name {
897 "RED" => Some(Color::Red),
898 "GREEN" => Some(Color::Green),
899 "BLUE" => Some(Color::Blue),
900 _ => None,
901 }
902 }
903 }
904
905 // ── test message ────────────────────────────────────────────────────────
906 //
907 // Hand-implemented to avoid depending on codegen in the runtime crate's
908 // own tests. Mirrors the pattern in `extension.rs` tests.
909
910 #[derive(Default, Clone, PartialEq, Debug)]
911 struct TestMsg {
912 i: i32,
913 s: String,
914 items: Vec<i64>,
915 child: Option<Box<TestMsg>>,
916 }
917
918 impl crate::DefaultInstance for TestMsg {
919 fn default_instance() -> &'static Self {
920 static INST: crate::__private::OnceBox<TestMsg> = crate::__private::OnceBox::new();
921 INST.get_or_init(|| Box::new(TestMsg::default()))
922 }
923 }
924
925 impl crate::Message for TestMsg {
926 fn compute_size(&self, _cache: &mut crate::SizeCache) -> u32 {
927 0
928 }
929 fn write_to(&self, _cache: &mut crate::SizeCache, _buf: &mut impl crate::EncodeSink) {}
930 fn merge_field(
931 &mut self,
932 tag: crate::encoding::Tag,
933 buf: &mut impl bytes::Buf,
934 _ctx: crate::DecodeContext<'_>,
935 ) -> Result<(), crate::DecodeError> {
936 crate::encoding::skip_field(tag, buf)
937 }
938 fn clear(&mut self) {
939 *self = Self::default();
940 }
941 }
942
943 impl TextFormat for TestMsg {
944 fn encode_text(&self, enc: &mut TextEncoder<'_>) -> core::fmt::Result {
945 if self.i != 0 {
946 enc.write_field_name("i")?;
947 enc.write_i32(self.i)?;
948 }
949 if !self.s.is_empty() {
950 enc.write_field_name("s")?;
951 enc.write_string(&self.s)?;
952 }
953 for &item in &self.items {
954 enc.write_field_name("items")?;
955 enc.write_i64(item)?;
956 }
957 if let Some(child) = &self.child {
958 enc.write_field_name("child")?;
959 enc.write_message(child.as_ref())?;
960 }
961 Ok(())
962 }
963
964 fn merge_text(&mut self, dec: &mut TextDecoder<'_>) -> Result<(), ParseError> {
965 while let Some(name) = dec.read_field_name()? {
966 match name {
967 "i" => self.i = dec.read_i32()?,
968 "s" => self.s = dec.read_string()?.into_owned(),
969 "items" => dec.read_repeated_scalar_into(&mut self.items, |d| d.read_i64())?,
970 "child" => {
971 let child = self.child.get_or_insert_with(Default::default);
972 dec.merge_message(child.as_mut())?;
973 }
974 _ => dec.skip_value()?,
975 }
976 }
977 Ok(())
978 }
979 }
980
981 // ── scalar read tables ──────────────────────────────────────────────────
982
983 #[test]
984 fn read_i32_table() {
985 #[rustfmt::skip]
986 let cases: &[(&str, Option<i32>)] = &[
987 ("42", Some(42)),
988 ("-7", Some(-7)),
989 ("0", Some(0)),
990 ("0x1F", Some(31)),
991 ("0X1f", Some(31)),
992 ("-0x10", Some(-16)),
993 ("0777", Some(511)),
994 ("-010", Some(-8)),
995 ("2147483647", Some(i32::MAX)),
996 ("-2147483648", Some(i32::MIN)),
997 // errors:
998 ("2147483648", None), // overflow
999 ("1.5", None), // float
1000 ("1f", None), // float suffix
1001 ];
1002 for &(input, want) in cases {
1003 // Wrap in a field so the tokenizer is in value-expecting state.
1004 let full = alloc::format!("f: {input}");
1005 let mut d = TextDecoder::new(&full);
1006 d.read_field_name().unwrap();
1007 assert_eq!(d.read_i32().ok(), want, "input: {input}");
1008 }
1009 }
1010
1011 #[test]
1012 fn read_u32_rejects_negative() {
1013 let mut d = TextDecoder::new("f: -1");
1014 d.read_field_name().unwrap();
1015 assert_eq!(d.read_u32().ok(), None);
1016 }
1017
1018 #[test]
1019 fn read_u64_table() {
1020 #[rustfmt::skip]
1021 let cases: &[(&str, Option<u64>)] = &[
1022 ("f: 0", Some(0)),
1023 ("f: 18446744073709551615", Some(u64::MAX)),
1024 ("f: 0xFFFFFFFFFFFFFFFF", Some(u64::MAX)),
1025 ("f: 18446744073709551616", None), // overflow
1026 ("f: -1", None), // negative
1027 ];
1028 for &(input, want) in cases {
1029 let mut d = TextDecoder::new(input);
1030 d.read_field_name().unwrap();
1031 assert_eq!(d.read_u64().ok(), want, "input: {input}");
1032 }
1033 }
1034
1035 #[test]
1036 fn read_f64_table() {
1037 // Floats can't use Option<f64> + == (NaN), so check classes.
1038 #[rustfmt::skip]
1039 let cases: &[(&str, &str)] = &[
1040 ("f: 1.5", "1.5"),
1041 ("f: -2.5", "-2.5"),
1042 ("f: .5", "0.5"),
1043 ("f: 1e3", "1000"),
1044 ("f: 1.5f", "1.5"), // f suffix stripped
1045 ("f: 42", "42"), // int → float
1046 ("f: 0", "0"), // plain zero (Dec, not Oct)
1047 ("f: 0.0", "0"),
1048 ("f: 0e0", "0"),
1049 ("f: inf", "inf"),
1050 ("f: -inf", "-inf"),
1051 ("f: - inf", "-inf"), // whitespace after sign
1052 ("f: -\tinf", "-inf"),
1053 ("f: - # c\ninf", "-inf"), // comment after sign
1054 ("f: -#c\n nan", "nan"),
1055 ("f: infinity", "inf"),
1056 ("f: Infinity", "inf"),
1057 ("f: -INFINITY", "-inf"),
1058 ("f: nan", "nan"),
1059 ("f: NaN", "nan"),
1060 ];
1061 for &(input, want) in cases {
1062 let mut d = TextDecoder::new(input);
1063 d.read_field_name().unwrap();
1064 let v = d
1065 .read_f64()
1066 .unwrap_or_else(|e| panic!("input {input}: {e}"));
1067 let got = if v.is_nan() {
1068 String::from("nan")
1069 } else if v.is_infinite() {
1070 String::from(if v > 0.0 { "inf" } else { "-inf" })
1071 } else {
1072 alloc::format!("{v}")
1073 };
1074 assert_eq!(got, want, "input: {input}");
1075 }
1076 }
1077
1078 #[test]
1079 fn read_f64_negative_zero() {
1080 // -0.0 must be preserved through the parse path. `f64::from_str`
1081 // handles this correctly for `"-0"`, `"-0.0"`, and underflow like
1082 // `"-1e-400"` — the sign bit survives. This test guards against
1083 // any future integer-parse-then-cast shortcut that would lose it.
1084 //
1085 // `-0.0 == 0.0` is true in IEEE float comparison, so assertions
1086 // must check the bit pattern (or sign) explicitly.
1087 #[rustfmt::skip]
1088 let cases: &[&str] = &[
1089 "-0", // NumKind::Dec
1090 "-0.0", // NumKind::Float
1091 "-0f", // NumKind::Float, f suffix
1092 "-0F", // NumKind::Float, F suffix
1093 "-0.0f",
1094 "-0e0", // NumKind::Float, exponent form
1095 "-1e-400", // underflow → -0.0
1096 "- 0", // whitespace between sign and digits
1097 ];
1098 for &input in cases {
1099 let full = alloc::format!("f: {input}");
1100 let mut d = TextDecoder::new(&full);
1101 d.read_field_name().unwrap();
1102 let v = d.read_f64().unwrap_or_else(|e| panic!("{input}: {e}"));
1103 assert!(
1104 v == 0.0 && v.is_sign_negative(),
1105 "{input}: want -0.0, got {v:?} (bits {:#018x})",
1106 v.to_bits()
1107 );
1108 }
1109 }
1110
1111 #[test]
1112 fn read_f32_negative_zero() {
1113 let mut d = TextDecoder::new("f: -0");
1114 d.read_field_name().unwrap();
1115 let v = d.read_f32().unwrap();
1116 assert!(v == 0.0 && v.is_sign_negative());
1117 assert_eq!(v.to_bits(), (-0.0f32).to_bits());
1118 }
1119
1120 #[test]
1121 fn read_f32_rounds_once() {
1122 // Each input rounds to a different `f32` when it is read as an `f64`
1123 // and narrowed: the `f64` lands on the midpoint of two `f32` values,
1124 // and the tie then goes the other way.
1125 let cases = [
1126 // Just past the midpoint of 1.0 and its neighbour.
1127 ("1.0000000596046447753906250000000000000001", 0x3f80_0001u32),
1128 (
1129 "-1.0000000596046447753906250000000000000001",
1130 0xbf80_0001u32,
1131 ),
1132 // The shortest decimal that identifies this `f32`.
1133 ("7.038531e-26", 0x15ae_43fd),
1134 // One below 2^128 - 2^103, the midpoint of `f32::MAX` and 2^128:
1135 // narrowed, it overflows to infinity.
1136 ("340282356779733661637539395458142568447", 0x7f7f_ffff),
1137 // Just above 2^-150, the midpoint of zero and the smallest
1138 // subnormal: narrowed, it underflows to zero.
1139 (
1140 "7.006492321624085354618647916449580656401309709382578858785341419448955413429303\
1141 00743319094181060791015625000000001e-46",
1142 0x0000_0001,
1143 ),
1144 ];
1145 for (input, expected) in cases {
1146 let narrowed = input.parse::<f64>().unwrap() as f32;
1147 assert_ne!(narrowed.to_bits(), expected, "input: {input}");
1148
1149 let full = alloc::format!("f: {input}");
1150 let mut d = TextDecoder::new(&full);
1151 d.read_field_name().unwrap();
1152 let got = d.read_f32().unwrap();
1153 assert_eq!(got.to_bits(), expected, "input: {input}");
1154 }
1155 }
1156
1157 #[test]
1158 fn read_f32_ties_overflow_underflow_and_suffix() {
1159 let cases = [
1160 // The exact midpoint of 1.0 and its neighbour ties to even.
1161 ("1.000000059604644775390625", 1.0f32),
1162 ("1e39", f32::INFINITY),
1163 ("-1e39", f32::NEG_INFINITY),
1164 ("1e-46", 0.0),
1165 ("-1e-46", -0.0),
1166 ("1.5f", 1.5),
1167 ("3.4028235e38", f32::MAX),
1168 ];
1169 for (input, expected) in cases {
1170 let full = alloc::format!("f: {input}");
1171 let mut d = TextDecoder::new(&full);
1172 d.read_field_name().unwrap();
1173 let got = d.read_f32().unwrap();
1174 assert_eq!(got.to_bits(), expected.to_bits(), "input: {input}");
1175 }
1176 }
1177
1178 #[test]
1179 fn read_f64_rejects_hex_and_octal() {
1180 // The textproto FLOAT grammar is base-10 only. Conformance tests
1181 // `FloatFieldNoHex`, `FloatFieldNoOctal` (and their negative forms)
1182 // require rejection.
1183 #[rustfmt::skip]
1184 let cases: &[&str] = &[
1185 "0x1", "-0x1", "0xFF",
1186 "01", "-01", "0777",
1187 ];
1188 for &input in cases {
1189 let full = alloc::format!("f: {input}");
1190 let mut d = TextDecoder::new(&full);
1191 d.read_field_name().unwrap();
1192 let err = d
1193 .read_f64()
1194 .expect_err(&alloc::format!("{input}: should reject hex/oct for float"));
1195 assert_eq!(err.kind, ParseErrorKind::InvalidNumber, "input: {input}");
1196 }
1197 // Sanity: these zero forms start with `0` but are valid floats
1198 // (Dec or Float, never Oct — lex_number only flags Oct when a
1199 // second octal digit follows).
1200 for &input in &["0", "0.0", "0e0", "0.", "0f"] {
1201 let full = alloc::format!("f: {input}");
1202 let mut d = TextDecoder::new(&full);
1203 d.read_field_name().unwrap();
1204 assert!(d.read_f64().is_ok(), "{input}: should be accepted");
1205 }
1206 }
1207
1208 #[test]
1209 fn read_f64_rejects_non_float_literal() {
1210 let mut d = TextDecoder::new("f: hello");
1211 d.read_field_name().unwrap();
1212 assert!(d.read_f64().is_err());
1213 }
1214
1215 #[test]
1216 fn read_bool_table() {
1217 #[rustfmt::skip]
1218 let cases: &[(&str, Option<bool>)] = &[
1219 ("f: true", Some(true)),
1220 ("f: True", Some(true)),
1221 ("f: t", Some(true)),
1222 ("f: false", Some(false)),
1223 ("f: False", Some(false)),
1224 ("f: f", Some(false)),
1225 ("f: 1", Some(true)),
1226 ("f: 0", Some(false)),
1227 ("f: 0x1", Some(true)),
1228 ("f: 01", Some(true)), // octal 1
1229 // errors:
1230 ("f: 2", None),
1231 ("f: yes", None),
1232 ("f: TRUE", None), // not in C++'s accepted set
1233 ];
1234 for &(input, want) in cases {
1235 let mut d = TextDecoder::new(input);
1236 d.read_field_name().unwrap();
1237 assert_eq!(d.read_bool().ok(), want, "input: {input}");
1238 }
1239 }
1240
1241 #[test]
1242 fn read_string_table() {
1243 #[rustfmt::skip]
1244 let cases: &[(&str, Option<&str>)] = &[
1245 (r#"f: "hello""#, Some("hello")),
1246 (r#"f: 'world'"#, Some("world")),
1247 (r#"f: "say \"hi\"""#, Some("say \"hi\"")),
1248 (r#"f: "foo" "bar""#, Some("foobar")),
1249 ("f: \"foo\" # c\n \"bar\"", Some("foobar")),
1250 (r#"f: """#, Some("")),
1251 (r#"f: 42"#, None), // not a string
1252 (r#"f: "\xFF""#, None), // invalid UTF-8
1253 ];
1254 for &(input, want) in cases {
1255 let mut d = TextDecoder::new(input);
1256 d.read_field_name().unwrap();
1257 let got = d.read_string().ok();
1258 assert_eq!(got.as_deref(), want, "input: {input}");
1259 }
1260 }
1261
1262 #[test]
1263 fn read_bytes_accepts_non_utf8() {
1264 let mut d = TextDecoder::new(r#"f: "\xFF\x00\x01""#);
1265 d.read_field_name().unwrap();
1266 assert_eq!(d.read_bytes().unwrap(), vec![0xFF, 0x00, 0x01]);
1267 }
1268
1269 #[test]
1270 fn read_enum_table() {
1271 #[rustfmt::skip]
1272 let cases: &[(&str, Option<i32>)] = &[
1273 ("f: RED", Some(0)),
1274 ("f: GREEN", Some(1)),
1275 ("f: BLUE", Some(2)),
1276 ("f: 0", Some(0)), // numeric form
1277 ("f: 99", Some(99)), // unknown number still OK (open enum)
1278 ("f: PURPLE", None), // unknown name → error
1279 ("f: -RED", None), // negative-prefixed name not an enum
1280 ];
1281 for &(input, want) in cases {
1282 let mut d = TextDecoder::new(input);
1283 d.read_field_name().unwrap();
1284 assert_eq!(d.read_enum_by_name::<Color>().ok(), want, "input: {input}");
1285 }
1286 }
1287
1288 #[test]
1289 fn read_enum_unknown_name_error_kind() {
1290 let mut d = TextDecoder::new("f: PURPLE");
1291 d.read_field_name().unwrap();
1292 let err = d.read_enum_by_name::<Color>().unwrap_err();
1293 assert_eq!(err.kind, ParseErrorKind::UnknownEnumValue);
1294 }
1295
1296 #[test]
1297 fn read_closed_enum_table() {
1298 // Closed enums (proto2 semantics) reject unknown NUMERIC values in
1299 // text format. Open enums accept them; see `read_enum_table` above
1300 // where `99` → `Some(99)`.
1301 #[rustfmt::skip]
1302 let cases: &[(&str, Option<Color>)] = &[
1303 ("f: RED", Some(Color::Red)),
1304 ("f: GREEN", Some(Color::Green)),
1305 ("f: 0", Some(Color::Red)), // numeric → known variant
1306 ("f: 2", Some(Color::Blue)),
1307 ("f: 99", None), // unknown number → error
1308 ("f: -1", None), // negative → error
1309 ("f: PURPLE", None), // unknown name → error (same as open)
1310 ];
1311 for &(input, want) in cases {
1312 let mut d = TextDecoder::new(input);
1313 d.read_field_name().unwrap();
1314 let got = d.read_closed_enum_by_name::<Color>().ok();
1315 assert_eq!(got, want, "input: {input}");
1316 }
1317 }
1318
1319 #[test]
1320 fn read_closed_enum_unknown_number_error_kind() {
1321 let mut d = TextDecoder::new("f: 99");
1322 d.read_field_name().unwrap();
1323 let err = d.read_closed_enum_by_name::<Color>().unwrap_err();
1324 assert_eq!(err.kind, ParseErrorKind::UnknownEnumValue);
1325 }
1326
1327 // ── repeated ────────────────────────────────────────────────────────────
1328
1329 #[test]
1330 fn repeated_both_forms() {
1331 // `items: 1 items: 2 items: [3, 4] items: 5` → [1,2,3,4,5]
1332 let mut m = TestMsg::default();
1333 let mut d = TextDecoder::new("items: 1 items: 2 items: [3, 4] items: 5");
1334 m.merge_text(&mut d).unwrap();
1335 assert_eq!(m.items, vec![1, 2, 3, 4, 5]);
1336 }
1337
1338 #[test]
1339 fn repeated_empty_list() {
1340 let mut m = TestMsg::default();
1341 let mut d = TextDecoder::new("items: []");
1342 m.merge_text(&mut d).unwrap();
1343 assert_eq!(m.items, Vec::<i64>::new());
1344 }
1345
1346 #[test]
1347 fn repeated_empty_scalar_list_requires_colon() {
1348 for input in ["items []", "i: 1 items []", "child { items [] }"] {
1349 let mut m = TestMsg::default();
1350 let err = m.merge_text(&mut TextDecoder::new(input)).unwrap_err();
1351 assert_eq!(
1352 err.kind,
1353 ParseErrorKind::UnexpectedToken {
1354 expected: "':' before scalar value",
1355 },
1356 "input: {input:?}"
1357 );
1358 assert_eq!(err.col as usize, input.find('[').unwrap() + 1);
1359 }
1360
1361 // With an element present the same error is positioned at the
1362 // element.
1363 for input in ["items [1]", "items 1"] {
1364 let mut m = TestMsg::default();
1365 let err = m.merge_text(&mut TextDecoder::new(input)).unwrap_err();
1366 assert_eq!(
1367 err.kind,
1368 ParseErrorKind::UnexpectedToken {
1369 expected: "':' before scalar value",
1370 },
1371 "input: {input:?}"
1372 );
1373 assert_eq!(err.col as usize, input.find('1').unwrap() + 1);
1374 }
1375
1376 // The colon that counts is the list field's own. One on the
1377 // enclosing field does not stand in for it, and its absence on the
1378 // enclosing field does not matter.
1379 let mut m = TestMsg::default();
1380 let err = m
1381 .merge_text(&mut TextDecoder::new("child: { items [] }"))
1382 .unwrap_err();
1383 assert_eq!(
1384 err.kind,
1385 ParseErrorKind::UnexpectedToken {
1386 expected: "':' before scalar value",
1387 }
1388 );
1389 for input in ["child { items: [] }", "child: { items: [] } items: []"] {
1390 let mut m = TestMsg::default();
1391 m.merge_text(&mut TextDecoder::new(input)).unwrap();
1392 assert_eq!(m.items, Vec::<i64>::new(), "input: {input:?}");
1393 }
1394
1395 // An empty scalar list with its colon, inside a message list that
1396 // was opened without one.
1397 let mut d = TextDecoder::new("f [{ items: [] }, { items [] }]");
1398 d.read_field_name().unwrap();
1399 let mut out: Vec<TestMsg> = Vec::new();
1400 let err = d
1401 .read_repeated_into(&mut out, |d| {
1402 let mut m = TestMsg::default();
1403 d.merge_message(&mut m)?;
1404 Ok(m)
1405 })
1406 .unwrap_err();
1407 assert_eq!(out.len(), 1, "the first element parses");
1408 assert_eq!(
1409 err.kind,
1410 ParseErrorKind::UnexpectedToken {
1411 expected: "':' before scalar value",
1412 }
1413 );
1414 }
1415
1416 #[test]
1417 fn repeated_message_list() {
1418 // `[{...}, {...}]` — exercises ListOpen → MessageOpen in the tokenizer.
1419 // TestMsg uses `child: Option<_>` not `Vec<_>`, so use a Vec<TestMsg>
1420 // directly through read_repeated_into.
1421 let mut d = TextDecoder::new("f: [{i: 1}, {i: 2}, <i: 3>]");
1422 d.read_field_name().unwrap();
1423 let mut out: Vec<TestMsg> = Vec::new();
1424 d.read_repeated_into(&mut out, |d| {
1425 let mut m = TestMsg::default();
1426 d.merge_message(&mut m)?;
1427 Ok(m)
1428 })
1429 .unwrap();
1430 assert_eq!(out.len(), 3);
1431 assert_eq!(out[0].i, 1);
1432 assert_eq!(out[1].i, 2);
1433 assert_eq!(out[2].i, 3);
1434 }
1435
1436 #[test]
1437 fn merge_from_str_appends() {
1438 use super::super::merge_from_str;
1439 let mut m = TestMsg {
1440 i: 1,
1441 items: vec![10],
1442 ..Default::default()
1443 };
1444 merge_from_str(&mut m, "i: 2 items: 20").unwrap();
1445 assert_eq!(m.i, 2); // scalar overwritten
1446 assert_eq!(m.items, vec![10, 20]); // repeated appended
1447 }
1448
1449 // ── roundtrip ───────────────────────────────────────────────────────────
1450
1451 #[test]
1452 fn roundtrip_simple() {
1453 let orig = TestMsg {
1454 i: 42,
1455 s: String::from("hello"),
1456 items: vec![1, 2, 3],
1457 child: None,
1458 };
1459 let text = encode_to_string(&orig);
1460 assert_eq!(text, r#"i: 42 s: "hello" items: 1 items: 2 items: 3"#);
1461 let back: TestMsg = decode_from_str(&text).unwrap();
1462 assert_eq!(back, orig);
1463 }
1464
1465 #[test]
1466 fn roundtrip_nested() {
1467 let orig = TestMsg {
1468 i: 1,
1469 s: String::new(),
1470 items: vec![],
1471 child: Some(Box::new(TestMsg {
1472 i: 2,
1473 s: String::from("inner"),
1474 items: vec![10, 20],
1475 child: None,
1476 })),
1477 };
1478 let text = encode_to_string(&orig);
1479 let back: TestMsg = decode_from_str(&text).unwrap();
1480 assert_eq!(back, orig);
1481 }
1482
1483 #[test]
1484 fn roundtrip_pretty() {
1485 let orig = TestMsg {
1486 i: 1,
1487 s: String::new(),
1488 items: vec![],
1489 child: Some(Box::new(TestMsg {
1490 i: 2,
1491 s: String::new(),
1492 items: vec![],
1493 child: None,
1494 })),
1495 };
1496 let text = encode_to_string_pretty(&orig);
1497 assert_eq!(text, "i: 1\nchild {\n i: 2\n}\n");
1498 let back: TestMsg = decode_from_str(&text).unwrap();
1499 assert_eq!(back, orig);
1500 }
1501
1502 #[test]
1503 fn parse_angle_delimiters() {
1504 // `<` `>` in place of `{` `}`.
1505 let m: TestMsg = decode_from_str("i: 1 child < i: 2 >").unwrap();
1506 assert_eq!(m.i, 1);
1507 assert_eq!(m.child.unwrap().i, 2);
1508 }
1509
1510 #[test]
1511 fn parse_canonical_cpp_output() {
1512 // What protoc --encode would produce for this message shape.
1513 let input = "i: 42\ns: \"hello\"\nitems: 1\nitems: 2\nchild {\n i: 7\n}\n";
1514 let m: TestMsg = decode_from_str(input).unwrap();
1515 assert_eq!(m.i, 42);
1516 assert_eq!(m.s, "hello");
1517 assert_eq!(m.items, vec![1, 2]);
1518 assert_eq!(m.child.unwrap().i, 7);
1519 }
1520
1521 #[test]
1522 fn parse_with_comments_and_separators() {
1523 let input = "# header\ni: 1, # inline\ns: \"x\"; # trailing\n";
1524 let m: TestMsg = decode_from_str(input).unwrap();
1525 assert_eq!(m.i, 1);
1526 assert_eq!(m.s, "x");
1527 }
1528
1529 // ── skip_value ──────────────────────────────────────────────────────────
1530
1531 #[test]
1532 fn skip_unknown_scalar() {
1533 let m: TestMsg = decode_from_str("unknown: 999 i: 42").unwrap();
1534 assert_eq!(m.i, 42);
1535 }
1536
1537 #[test]
1538 fn skip_unknown_message() {
1539 let m: TestMsg = decode_from_str("unknown { x: 1 y: 2 } i: 42").unwrap();
1540 assert_eq!(m.i, 42);
1541 }
1542
1543 #[test]
1544 fn skip_unknown_nested_message() {
1545 let m: TestMsg = decode_from_str("unknown { inner { deep: 1 } } i: 42").unwrap();
1546 assert_eq!(m.i, 42);
1547 }
1548
1549 #[test]
1550 fn skip_unknown_list() {
1551 let m: TestMsg = decode_from_str("unknown: [1, 2, 3] i: 42").unwrap();
1552 assert_eq!(m.i, 42);
1553 }
1554
1555 #[test]
1556 fn skip_unknown_message_list() {
1557 let m: TestMsg = decode_from_str("unknown: [{a: 1}, {a: 2}] i: 42").unwrap();
1558 assert_eq!(m.i, 42);
1559 }
1560
1561 #[test]
1562 fn skip_unknown_list_without_colon() {
1563 let m: TestMsg = decode_from_str("unknown [{a: 1}, {a: 2}] i: 42").unwrap();
1564 assert_eq!(m.i, 42);
1565 let err = decode_from_str::<TestMsg>("unknown [1, 2] i: 42").unwrap_err();
1566 assert_eq!(
1567 err.kind,
1568 ParseErrorKind::UnexpectedToken {
1569 expected: "':' before scalar value",
1570 }
1571 );
1572 }
1573
1574 // ── errors ──────────────────────────────────────────────────────────────
1575
1576 #[test]
1577 fn unknown_field_error() {
1578 // A merge_text impl that doesn't skip unknowns.
1579 #[derive(Default, Clone, PartialEq, Debug)]
1580 struct Strict {
1581 i: i32,
1582 }
1583 impl crate::DefaultInstance for Strict {
1584 fn default_instance() -> &'static Self {
1585 static I: crate::__private::OnceBox<Strict> = crate::__private::OnceBox::new();
1586 I.get_or_init(|| Box::new(Strict::default()))
1587 }
1588 }
1589 impl crate::Message for Strict {
1590 fn compute_size(&self, _: &mut crate::SizeCache) -> u32 {
1591 0
1592 }
1593 fn write_to(&self, _: &mut crate::SizeCache, _: &mut impl crate::EncodeSink) {}
1594 fn merge_field(
1595 &mut self,
1596 t: crate::encoding::Tag,
1597 b: &mut impl bytes::Buf,
1598 _: crate::DecodeContext<'_>,
1599 ) -> Result<(), crate::DecodeError> {
1600 crate::encoding::skip_field(t, b)
1601 }
1602 fn clear(&mut self) {}
1603 }
1604 impl TextFormat for Strict {
1605 fn encode_text(&self, _: &mut TextEncoder<'_>) -> core::fmt::Result {
1606 Ok(())
1607 }
1608 fn merge_text(&mut self, d: &mut TextDecoder<'_>) -> Result<(), ParseError> {
1609 while let Some(name) = d.read_field_name()? {
1610 match name {
1611 "i" => self.i = d.read_i32()?,
1612 _ => return Err(d.unknown_field()),
1613 }
1614 }
1615 Ok(())
1616 }
1617 }
1618
1619 let err = decode_from_str::<Strict>("i: 1\nbad: 2").unwrap_err();
1620 assert_eq!(err.kind, ParseErrorKind::UnknownField);
1621 assert_eq!(err.line, 2);
1622 assert_eq!(err.col, 1);
1623 }
1624
1625 #[test]
1626 fn depth_limit_exceeded() {
1627 // Build a chain of `child { child { ... } }` past the limit.
1628 let depth = crate::message::RECURSION_LIMIT as usize + 1;
1629 let mut s = String::new();
1630 for _ in 0..depth {
1631 s.push_str("child { ");
1632 }
1633 for _ in 0..depth {
1634 s.push('}');
1635 }
1636 let err = decode_from_str::<TestMsg>(&s).unwrap_err();
1637 assert_eq!(err.kind, ParseErrorKind::RecursionLimitExceeded);
1638 }
1639
1640 #[test]
1641 fn error_position_in_value() {
1642 // Error on line 2, in the value.
1643 let err = decode_from_str::<TestMsg>("i: 1\ni: notanumber").unwrap_err();
1644 assert_eq!(err.line, 2);
1645 assert!(matches!(err.kind, ParseErrorKind::UnexpectedToken { .. }));
1646 }
1647}