1use crate::error::{Error, ErrorCode, Result};
2use alloc::vec::Vec;
3use core::cmp;
4use core::mem;
5use core::ops::Deref;
6use core::str;
78#[cfg(feature = "std")]
9use crate::io;
1011#[cfg(feature = "raw_value")]
12use crate::raw::BorrowedRawDeserializer;
13#[cfg(all(feature = "raw_value", feature = "std"))]
14use crate::raw::OwnedRawDeserializer;
15#[cfg(all(feature = "raw_value", feature = "std"))]
16use alloc::string::String;
17#[cfg(feature = "raw_value")]
18use serde::de::Visitor;
1920/// Trait used by the deserializer for iterating over input. This is manually
21/// "specialized" for iterating over `&[u8]`. Once feature(specialization) is
22/// stable we can use actual specialization.
23///
24/// This trait is sealed and cannot be implemented for types outside of
25/// `serde_json`.
26pub trait Read<'de>: private::Sealed {
27#[doc(hidden)]
28fn next(&mut self) -> Result<Option<u8>>;
29#[doc(hidden)]
30fn peek(&mut self) -> Result<Option<u8>>;
3132/// Only valid after a call to peek(). Discards the peeked byte.
33#[doc(hidden)]
34fn discard(&mut self);
3536/// Skip insignificant whitespace and peek the next byte, without consuming
37 /// it. Equivalent to `while peek() is whitespace { discard() }; peek()`.
38 ///
39 /// This exists as its own method because whitespace is the single largest
40 /// class of byte in pretty-printed JSON -- 71.9% of `citm_catalog.json`,
41 /// where skipping it measured **36% of the time to parse the document into
42 /// a `Value`**. Going through `peek()`/`discard()` costs a `Result<Option>`
43 /// construction, a match and a bounds check for every one of those bytes.
44 /// A reader that owns a contiguous buffer can do far better, so it gets to
45 /// override this; the default is exactly the loop it replaces.
46 ///
47 /// Only ` `, `\n`, `\t` and `\r` are insignificant. Any other byte <= 0x20
48 /// is invalid at this position and must be RETURNED rather than skipped, so
49 /// that the parser reports it at the right line and column.
50#[doc(hidden)]
51 #[inline]
52fn skip_whitespace(&mut self) -> Result<Option<u8>> {
53loop {
54match match self.peek() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(self.peek()) {
55Some(b' ' | b'\n' | b'\t' | b'\r') => self.discard(),
56 other => return Ok(other),
57 }
58 }
59 }
6061/// If the next eight bytes are all ASCII digits, consume them and return
62 /// their value; otherwise consume nothing and return `None`.
63 ///
64 /// The number path is the whitespace path all over again: `canada.json` is
65 /// **90.1% number bytes** -- 2.03 MB of digits across 111,126 numbers,
66 /// about 17 digits each -- and every one of them goes through a
67 /// `peek_or_null()`, a range test, an overflow check and an `eat_char()`.
68 /// After the whitespace work that file still made 2.19 million `peek` and
69 /// 2.14 million `discard` calls, essentially all of them here.
70 ///
71 /// The caller must only use this while an overflow is impossible for the
72 /// whole chunk, so that the per-digit overflow check it replaces could not
73 /// have fired either. The default implementation declines, which is correct
74 /// for any reader that is not backed by a contiguous buffer.
75#[doc(hidden)]
76 #[inline]
77fn take_8_digits(&mut self) -> Option<u32> {
78None79 }
8081/// Position of the most recent call to next().
82 ///
83 /// The most recent call was probably next() and not peek(), but this method
84 /// should try to return a sensible result if the most recent call was
85 /// actually peek() because we don't always know.
86 ///
87 /// Only called in case of an error, so performance is not important.
88#[doc(hidden)]
89fn position(&self) -> Position;
9091/// Position of the most recent call to peek().
92 ///
93 /// The most recent call was probably peek() and not next(), but this method
94 /// should try to return a sensible result if the most recent call was
95 /// actually next() because we don't always know.
96 ///
97 /// Only called in case of an error, so performance is not important.
98#[doc(hidden)]
99fn peek_position(&self) -> Position;
100101/// Offset from the beginning of the input to the next byte that would be
102 /// returned by next() or peek().
103#[doc(hidden)]
104fn byte_offset(&self) -> usize;
105106/// Assumes the previous byte was a quotation mark. Parses a JSON-escaped
107 /// string until the next quotation mark using the given scratch space if
108 /// necessary. The scratch space is initially empty.
109#[doc(hidden)]
110fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'de, 's, str>>;
111112/// Assumes the previous byte was a quotation mark. Parses a JSON-escaped
113 /// string until the next quotation mark using the given scratch space if
114 /// necessary. The scratch space is initially empty.
115 ///
116 /// This function returns the raw bytes in the string with escape sequences
117 /// expanded but without performing unicode validation.
118#[doc(hidden)]
119fn parse_str_raw<'s>(
120&'s mut self,
121 scratch: &'s mut Vec<u8>,
122 ) -> Result<Reference<'de, 's, [u8]>>;
123124/// Assumes the previous byte was a quotation mark. Parses a JSON-escaped
125 /// string until the next quotation mark but discards the data.
126#[doc(hidden)]
127fn ignore_str(&mut self) -> Result<()>;
128129/// Assumes the previous byte was a hex escape sequence ('\u') in a string.
130 /// Parses next hexadecimal sequence.
131#[doc(hidden)]
132fn decode_hex_escape(&mut self) -> Result<u16>;
133134/// Switch raw buffering mode on.
135 ///
136 /// This is used when deserializing `RawValue`.
137#[cfg(feature = "raw_value")]
138 #[doc(hidden)]
139fn begin_raw_buffering(&mut self);
140141/// Switch raw buffering mode off and provides the raw buffered data to the
142 /// given visitor.
143#[cfg(feature = "raw_value")]
144 #[doc(hidden)]
145fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
146where
147V: Visitor<'de>;
148149/// Whether StreamDeserializer::next needs to check the failed flag. True
150 /// for IoRead, false for StrRead and SliceRead which can track failure by
151 /// truncating their input slice to avoid the extra check on every next
152 /// call.
153#[doc(hidden)]
154const should_early_return_if_failed: bool;
155156/// Mark a persistent failure of StreamDeserializer, either by setting the
157 /// flag or by truncating the input data.
158#[doc(hidden)]
159fn set_failed(&mut self, failed: &mut bool);
160}
161162pub struct Position {
163pub line: usize,
164pub column: usize,
165}
166167pub enum Reference<'b, 'c, T>
168where
169T: ?Sized + 'static,
170{
171 Borrowed(&'b T),
172 Copied(&'c T),
173}
174175impl<'b, 'c, T> Dereffor Reference<'b, 'c, T>
176where
177T: ?Sized + 'static,
178{
179type Target = T;
180181fn deref(&self) -> &Self::Target {
182match *self {
183 Reference::Borrowed(b) => b,
184 Reference::Copied(c) => c,
185 }
186 }
187}
188189/// JSON input source that reads from a std::io input stream.
190#[cfg(feature = "std")]
191#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
192/// First byte at or after `from` that is not JSON whitespace.
193///
194/// The island when it is linked, and a plain walk when it is not, so a
195/// `--no-default-features` build still compiles and still behaves identically.
196#[cfg(feature = "std")]
197#[inline]
198fn first_non_ws_in(slice: &[u8], from: usize) -> usize {
199#[cfg(feature = "accel")]
200{
201 rusty_json_turbo_accel::first_non_ws(slice, from)
202 }
203// Without the island, the crate's OWN 8-byte SWAR -- not a byte walk. The
204 // island is opt-in now, so this is the path that actually ships, and
205 // handing the reader a scalar loop here would throw away most of what
206 // brick B7 bought.
207#[cfg(not(feature = "accel"))]
208{
209let mut i = from;
210while let Some(window) = slice.get(i..i + 8) {
211let Ok(eight) = <[u8; 8]>::try_from(window) else {
212break;
213 };
214let m = non_ws_bytes(u64::from_le_bytes(eight));
215if m != 0 {
216return i + (m.trailing_zeros() >> 3) as usize;
217 }
218 i += 8;
219 }
220while i < slice.len() && is_ws(slice[i]) {
221 i += 1;
222 }
223i224 }
225}
226227/// Bytes pulled from the reader per refill. Big enough that the per-call cost
228/// disappears, small enough to stay in L1 alongside everything else.
229#[cfg(feature = "std")]
230const IO_CHUNK: usize = 8192;
231232/// JSON input source that reads from a std::io input stream.
233///
234/// BRICK B7. Upstream held a `LineColIterator<io::Bytes<R>>` -- one
235/// `io::Result<u8>` per byte, wrapped in a per-byte line and column
236/// counter. Measured, `from_reader` was 1.55x slower than `from_slice` on
237/// `citm_catalog` and 1.28x on `twitter`, and the gap tracked WHITESPACE
238/// exactly, because every whitespace brick this crate has landed works on
239/// a slice and this reader had none to offer.
240///
241/// Now it holds a window. The bytes are a slice, so the same scanners can
242/// run over them, and the line and column counters are computed only when
243/// an error actually needs them.
244#[cfg(feature = "std")]
245#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
246pub struct IoRead<R>
247where
248R: io::Read,
249{
250 reader: R,
251/// The window, allocated ONCE and never resized.
252 ///
253 /// An earlier version grew it with `resize(len + CHUNK, 0)` per refill,
254 /// which zeroed eight kilobytes every time -- as much zeroing as parsing
255 /// on a document of any size. `filled` marks how much is valid instead.
256buf: Vec<u8>,
257/// Bytes of `buf` holding real data; `buf[pos..filled]` is unread.
258filled: usize,
259 pos: usize,
260/// Absolute stream offset of `buf[0]`.
261base: usize,
262/// The reader has returned zero bytes.
263eof: bool,
264/// The peeked byte, held in a register the way upstream held it.
265 ///
266 /// This is not redundant with the buffer, and an earlier version of this
267 /// brick learned that the expensive way. Serving `peek` straight out of
268 /// the buffer means a length compare, a bounds-checked index and a store
269 /// on EVERY call -- and `canada.json` makes 2.2 million of them. That
270 /// version regressed `canada` from 1.10x to 1.32x behind `from_slice`
271 /// while improving `citm_catalog`, which is not a trade this project
272 /// accepts.
273 ///
274 /// So the hot path stays exactly upstream's shape -- one `Option` in a
275 /// register -- and the buffer sits underneath it, where the wide scanners
276 /// can reach the bytes. `pos` is already past whatever `ch` holds, which
277 /// is also what makes the position arithmetic below identical to
278 /// upstream's.
279ch: Option<u8>,
280/// Line number of the line containing `base`, 1-based.
281line_at_base: usize,
282/// Absolute offset of the first byte of the line containing `base`.
283sol_at_base: usize,
284/// `Some` while a `RawValue` is being captured. BRICK B13.
285 ///
286 /// It stays EMPTY: the captured bytes are taken from the window in one
287 /// copy at `end_raw_buffering`, and this is only the flag that says a
288 /// capture is running -- kept as a `Vec` so that a nested capture behaves
289 /// exactly as upstream's did.
290#[cfg(feature = "raw_value")]
291raw_buffer: Option<Vec<u8>>,
292/// Index in `buf` of the first byte of the value being captured.
293#[cfg(feature = "raw_value")]
294raw_from: usize,
295}
296297/// JSON input source that reads from a slice of bytes.
298//
299// This is more efficient than other iterators because peek() can be read-only
300// and we can compute line/col position only if an error happens.
301pub struct SliceRead<'a> {
302 slice: &'a [u8],
303/// Index of the *next* byte that will be returned by next() or peek().
304index: usize,
305#[cfg(feature = "raw_value")]
306raw_buffering_start_index: usize,
307}
308309/// JSON input source that reads from a UTF-8 string.
310//
311// Able to elide UTF-8 checks by assuming that the input is valid UTF-8.
312pub struct StrRead<'a> {
313 delegate: SliceRead<'a>,
314#[cfg(feature = "raw_value")]
315data: &'a str,
316}
317318// Prevent users from implementing the Read trait.
319mod private {
320pub trait Sealed {}
321}
322323//////////////////////////////////////////////////////////////////////////////
324325#[cfg(feature = "std")]
326impl<R> IoRead<R>
327where
328R: io::Read,
329{
330/// Create a JSON input source to read from a std::io input stream.
331 ///
332 /// **This buffers internally, so you do not need a
333 /// [`std::io::BufReader`].** Upstream did not, and told you to add one;
334 /// this reads in 8 KiB blocks itself, so wrapping the input in a second
335 /// buffer only adds a layer. It is harmless on a [`File`] and a measurable
336 /// loss on data already in memory.
337 ///
338 /// For bytes you already hold, [`from_slice`](crate::from_slice) is still
339 /// the faster route -- it can borrow strings straight out of the input,
340 /// which no reader can do.
341 ///
342 /// [`File`]: std::fs::File
343pub fn new(reader: R) -> Self {
344IoRead {
345reader,
346 buf: ::alloc::vec::from_elem(0, IO_CHUNK * 2)alloc::vec![0; IO_CHUNK * 2],
347 filled: 0,
348 pos: 0,
349 base: 0,
350 eof: false,
351 ch: None,
352 line_at_base: 1,
353 sol_at_base: 0,
354#[cfg(feature = "raw_value")]
355raw_buffer: None,
356#[cfg(feature = "raw_value")]
357raw_from: 0,
358 }
359 }
360361/// Drop what has been consumed and pull more bytes.
362 ///
363 /// Only called when the window is exhausted, so nothing peeked is ever
364 /// dropped: a peeked byte is by definition still in the window.
365// COLD, and marked so deliberately: a refill happens once per 8 KiB, so
366 // inlining it into `pull` would put its `memchr`, its `copy_within` and its
367 // `read` call in the middle of a path taken once per BYTE. `canada.json`
368 // makes 2.2 million of those.
369#[cold]
370 #[inline(never)]
371fn fill(&mut self) -> Result<()> {
372// BRICK B13. While a `RawValue` is being captured its bytes have to
373 // stay addressable, so the window is drained only as far as where the
374 // capture STARTS rather than as far as `pos`. Everything before
375 // `raw_from` is consumed and unreachable, so dropping that much is
376 // free, and what is held is bounded by the size of the value -- which
377 // is exactly what upstream's raw buffer held anyway.
378#[cfg(feature = "raw_value")]
379let drop_to = if self.raw_buffer.is_some() {
380// The whole prefix before the capture goes, so `raw_from` lands
381 // on 0 -- which `take` does in the same move as reading it.
382core::mem::take(&mut self.raw_from)
383 } else {
384self.pos
385 };
386#[cfg(not(feature = "raw_value"))]
387let drop_to = self.pos;
388389if drop_to > 0 {
390// The bytes about to be dropped are the only place the line
391 // counters can be advanced from, so roll them forward first.
392self.roll_lines(drop_to);
393self.buf.copy_within(drop_to..self.filled, 0);
394self.filled -= drop_to;
395self.base += drop_to;
396self.pos -= drop_to;
397 }
398if self.eof {
399return Ok(());
400 }
401if self.filled == self.buf.len() {
402// A full window with nothing droppable is only reachable while
403 // capturing a `RawValue` whose text is longer than the buffer:
404 // otherwise the drain above frees the whole consumed prefix, and
405 // `fill` is only ever called with the window spent. Grow instead
406 // of sliding, so the captured bytes stay put; doubling keeps the
407 // number of growths logarithmic in the size of the value.
408#[cfg(feature = "raw_value")]
409{
410if true {
if !self.raw_buffer.is_some() {
{
::core::panicking::panic_fmt(format_args!("the window can only be full with nothing to drop while capturing"));
}
};
};debug_assert!(
411self.raw_buffer.is_some(),
412"the window can only be full with nothing to drop while capturing"
413);
414crate::counters::add(&crate::counters::RAW_GROWS, 1);
415self.buf.resize(self.buf.len() * 2, 0);
416 }
417#[cfg(not(feature = "raw_value"))]
418return Ok(());
419 }
420// A short read is fine and expected from a pipe or socket: the caller
421 // asks again, and `eof` is set only by a read of length zero.
422match self.reader.read(&mut self.buf[self.filled..]) {
423Ok(0) => {
424self.eof = true;
425Ok(())
426 }
427Ok(n) => {
428self.filled += n;
429Ok(())
430 }
431Err(e) => Err(Error::io(e)),
432 }
433 }
434435/// Advance `line_at_base` / `sol_at_base` over `buf[..n]`.
436 ///
437 /// `memchr` rather than a per-byte branch, which is the other half of what
438 /// B7 removes: upstream counted lines on every byte whether or not any
439 /// error was ever reported.
440fn roll_lines(&mut self, n: usize) {
441let seg = &self.buf[..n];
442let mut count = 0;
443let mut last = None;
444for i in memchr::memchr_iter(b'\n', seg) {
445 count += 1;
446 last = Some(i);
447 }
448if let Some(i) = last {
449self.sol_at_base = self.base + i + 1;
450 }
451self.line_at_base += count;
452 }
453454/// Line and column at absolute offset `k`, matching `LineColIterator`
455 /// exactly: line is 1-based and counts newlines PULLED, column is the
456 /// number of bytes since the start of the current line, so it is 0
457 /// immediately after a newline.
458 ///
459 /// Only reached from an error path, so scanning the window here is free in
460 /// the sense that matters.
461fn position_at(&self, k: usize) -> Position {
462let rel = (k - self.base).min(self.filled);
463let mut line = self.line_at_base;
464let mut sol = self.sol_at_base;
465for i in memchr::memchr_iter(b'\n', &self.buf[..rel]) {
466 line += 1;
467 sol = self.base + i + 1;
468 }
469Position {
470line,
471 column: k - sol,
472 }
473 }
474475/// Bytes PULLED from the stream. `pos` is already past a byte held in
476 /// `ch`, so this is exactly what `LineColIterator::byte_offset` reported.
477fn pulled(&self) -> usize {
478self.base + self.pos
479 }
480481/// Pull one byte into `ch`, refilling if the window is spent.
482#[inline(always)]
483fn pull(&mut self) -> Result<Option<u8>> {
484if self.pos == self.filled {
485match self.fill() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(self.fill());
486if self.pos == self.filled {
487return Ok(None);
488 }
489 }
490let ch = self.buf[self.pos];
491self.pos += 1;
492Ok(Some(ch))
493 }
494}
495496#[cfg(feature = "std")]
497impl<R> private::Sealedfor IoRead<R> where R: io::Read {}
498499#[cfg(feature = "std")]
500impl<R> IoRead<R>
501where
502R: io::Read,
503{
504fn parse_str_bytes<'s, T, F>(
505&'s mut self,
506 scratch: &'s mut Vec<u8>,
507 validate: bool,
508 result: F,
509 ) -> Result<T>
510where
511T: 's,
512 F: FnOnce(&'s Self, &'s [u8]) -> Result<T>,
513 {
514loop {
515let ch = match next_or_eof(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
516if !is_escape(ch, true) {
517scratch.push(ch);
518continue;
519 }
520match ch {
521b'"' => {
522return result(self, scratch);
523 }
524b'\\' => {
525match parse_escape(self, validate, scratch) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(parse_escape(self, validate, scratch));
526 }
527_ => {
528if validate {
529return error(self, ErrorCode::ControlCharacterWhileParsingString);
530 }
531scratch.push(ch);
532 }
533 }
534 }
535 }
536}
537538#[cfg(feature = "std")]
539impl<'de, R> Read<'de> for IoRead<R>
540where
541R: io::Read,
542{
543#[inline]
544fn next(&mut self) -> Result<Option<u8>> {
545// No `RawValue` push here: brick B13 takes the captured text as one
546 // range from the window, so this is the same two instructions whether
547 // or not the `raw_value` feature is on.
548match self.ch.take() {
549Some(ch) => Ok(Some(ch)),
550None => self.pull(),
551 }
552 }
553554#[inline]
555fn peek(&mut self) -> Result<Option<u8>> {
556match self.ch {
557Some(ch) => Ok(Some(ch)),
558None => {
559self.ch = match self.pull() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(self.pull());
560Ok(self.ch)
561 }
562 }
563 }
564565#[inline]
566fn discard(&mut self) {
567self.ch = None;
568 }
569570/// THE POINT OF BRICK B7: run the wide whitespace scanner over the window.
571 ///
572 /// This is the half of the ceiling the buffer exists to unlock. Measured,
573 /// the 1.55x gap between `from_reader` and `from_slice` on `citm_catalog`
574 /// decomposed as roughly 1.41x from the missing slice scanners and only
575 /// 1.10x from the per-byte iterator -- so a buffer WITHOUT this override is
576 /// all cost and no benefit, which an earlier version of this brick duly
577 /// measured as a regression on every file.
578 ///
579 /// A whitespace run may cross a refill, so the scan repeats until it finds
580 /// a non-whitespace byte or the reader is spent. The byte is left in `ch`
581 /// rather than consumed, exactly as the trait's `peek`-based default leaves
582 /// it, so error positions and `RawValue` capture are unchanged.
583#[inline]
584fn skip_whitespace(&mut self) -> Result<Option<u8>> {
585// Whatever was already peeked comes first, or the scan would step over
586 // it and lose a byte.
587if let Some(c) = self.ch {
588if !is_ws(c) {
589return Ok(Some(c));
590 }
591self.ch = None;
592 }
593// THE TINY ENTRY, and this is the third time this project has needed
594 // one. On a minified document there is no whitespace at all, so every
595 // call would otherwise enter the scanner, dispatch on the ISA, and
596 // find its answer at offset zero. `canada.json` and
597 // `s4-frame-telemetry.json` regressed 1.10x -> 1.16x and 1.02x ->
598 // 1.10x on exactly that before this existed.
599 //
600 // Brick B1s needed a 4-byte scalar peel for the same reason, and M3's
601 // escape scanner needed a length guard. The law is the same each time:
602 // a wide scanner must not be reached until something cheap has ruled
603 // out the common case.
604if self.pos < self.filled {
605let c = self.buf[self.pos];
606if !is_ws(c) {
607self.pos += 1;
608self.ch = Some(c);
609return Ok(Some(c));
610 }
611 }
612loop {
613let at = first_non_ws_in(&self.buf[..self.filled], self.pos);
614// A `RawValue` captures the raw TEXT of a value, whitespace and
615 // all -- `{"foo": 2}` must not come back as `{"foo":2}`, which is
616 // what upstream's `test_boxed_raw_value` caught when an earlier
617 // version of this override skipped a run without recording it.
618 // Nothing is recorded here any more: brick B13 takes the whole
619 // range at the end, whitespace included.
620if at < self.filled {
621// Pull it: `pos` moves past the byte and `ch` holds it, which
622 // is the state a `peek` would have left.
623self.pos = at + 1;
624let c = self.buf[at];
625self.ch = Some(c);
626return Ok(Some(c));
627 }
628self.pos = self.filled;
629match self.fill() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(self.fill());
630if self.pos == self.filled {
631return Ok(None);
632 }
633 }
634 }
635636fn position(&self) -> Position {
637self.position_at(self.pulled())
638 }
639640fn peek_position(&self) -> Position {
641// Upstream's comment: "The LineColIterator updates its position during
642 // peek() so it has the right one here." A peek here pulls too, so the
643 // same is true and these stay equal.
644self.position()
645 }
646647fn byte_offset(&self) -> usize {
648// Upstream computed `iter.byte_offset() - 1` when a byte was peeked.
649 // Same arithmetic, same meaning: bytes CONSUMED.
650match self.ch {
651Some(_) => self.pulled() - 1,
652None => self.pulled(),
653 }
654 }
655656fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'de, 's, str>> {
657self.parse_str_bytes(scratch, true, as_str)
658 .map(Reference::Copied)
659 }
660661fn parse_str_raw<'s>(
662&'s mut self,
663 scratch: &'s mut Vec<u8>,
664 ) -> Result<Reference<'de, 's, [u8]>> {
665self.parse_str_bytes(scratch, false, |_, bytes| Ok(bytes))
666 .map(Reference::Copied)
667 }
668669fn ignore_str(&mut self) -> Result<()> {
670loop {
671let ch = match next_or_eof(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
672if !is_escape(ch, true) {
673continue;
674 }
675match ch {
676b'"' => {
677return Ok(());
678 }
679b'\\' => {
680match ignore_escape(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(ignore_escape(self));
681 }
682_ => {
683return error(self, ErrorCode::ControlCharacterWhileParsingString);
684 }
685 }
686 }
687 }
688689fn decode_hex_escape(&mut self) -> Result<u16> {
690let a = match next_or_eof(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
691let b = match next_or_eof(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
692let c = match next_or_eof(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
693let d = match next_or_eof(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
694match decode_four_hex_digits(a, b, c, d) {
695Some(val) => Ok(val),
696None => error(self, ErrorCode::InvalidEscape),
697 }
698 }
699700/// BRICK B13: a `RawValue` over a reader as a RANGE, not a byte stream.
701 ///
702 /// Upstream captured a `RawValue` by pushing every byte into a `Vec` as it
703 /// was consumed, through `next` and `discard`. Priced on a 10,000-document
704 /// log stream, `RawValue` over a reader ran at 257 MB/s against **1,074
705 /// MB/s for the same values over a slice** -- 4.18x -- and sat at parity
706 /// with upstream (1.009x), because the per-byte push, not the parse, was
707 /// the cost.
708 ///
709 /// The bytes are already contiguous in the window, so all this has to
710 /// record is where the value starts; `fill` then holds the window from
711 /// there and `end_raw_buffering` takes it in one copy.
712 ///
713 /// A byte sitting in `ch` has been PULLED but not CONSUMED, and upstream
714 /// pushed it only if it was later consumed -- so the start is `pos` less
715 /// that byte, which is the same arithmetic as `byte_offset`.
716#[cfg(feature = "raw_value")]
717fn begin_raw_buffering(&mut self) {
718self.raw_from = self.pos - usize::from(self.ch.is_some());
719self.raw_buffer = Some(Vec::new());
720 }
721722#[cfg(feature = "raw_value")]
723fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
724where
725V: Visitor<'de>,
726 {
727// Whatever is in `ch` was peeked to END the value and is not part of
728 // it -- a number stops on the byte after its last digit.
729let end = self.pos - usize::from(self.ch.is_some());
730let mut raw = self.raw_buffer.take().unwrap();
731// ONE copy, of exactly the right size: `extend_from_slice` on an empty
732 // `Vec` reserves the slice's length, and `String::from_utf8` then
733 // takes the allocation as it stands.
734crate::counters::add(&crate::counters::RAW_CAPTURES, 1);
735crate::counters::add(&crate::counters::RAW_BYTES, (end - self.raw_from) as u64);
736raw.extend_from_slice(&self.buf[self.raw_from..end]);
737// Give any growth back. A value longer than the window grew `buf`,
738 // and a stream carrying one enormous value must not go on paying for
739 // it on every document after -- upstream allocated a fresh `Vec` per
740 // value and freed it each time, and this has to be no worse. In the
741 // common case the window never grew and this is one length compare.
742if self.buf.len() > IO_CHUNK * 2 && self.filled - self.pos <= IO_CHUNK * 2 {
743self.roll_lines(self.pos);
744self.buf.copy_within(self.pos..self.filled, 0);
745self.filled -= self.pos;
746self.base += self.pos;
747self.pos = 0;
748self.buf.truncate(IO_CHUNK * 2);
749self.buf.shrink_to_fit();
750crate::counters::add(&crate::counters::RAW_SHRINKS, 1);
751 }
752let raw = match String::from_utf8(raw) {
753Ok(raw) => raw,
754Err(_) => return error(self, ErrorCode::InvalidUnicodeCodePoint),
755 };
756visitor.visit_map(OwnedRawDeserializer {
757 raw_value: Some(raw),
758 })
759 }
760761const should_early_return_if_failed: bool = true;
762763#[inline]
764 #[cold]
765fn set_failed(&mut self, failed: &mut bool) {
766*failed = true;
767 }
768}
769770//////////////////////////////////////////////////////////////////////////////
771772impl<'a> SliceRead<'a> {
773/// Create a JSON input source to read from a slice of bytes.
774pub fn new(slice: &'a [u8]) -> Self {
775SliceRead {
776slice,
777 index: 0,
778#[cfg(feature = "raw_value")]
779raw_buffering_start_index: 0,
780 }
781 }
782783fn position_of_index(&self, i: usize) -> Position {
784let start_of_line = match memchr::memrchr(b'\n', &self.slice[..i]) {
785Some(position) => position + 1,
786None => 0,
787 };
788Position {
789 line: 1 + memchr::memchr_iter(b'\n', &self.slice[..start_of_line]).count(),
790 column: i - start_of_line,
791 }
792 }
793794/// PROBE WRAPPER (brick B2). Counts one run and the bytes it advanced
795 /// over, for every caller, without touching the scanner itself.
796 /// `inline(always)` on a body that compiles to nothing without `profile`,
797 /// so a shipping build is byte-for-byte what it was.
798#[inline(always)]
799fn skip_to_escape(&mut self, forbid_control_characters: bool) {
800let from = self.index;
801self.skip_to_escape_impl(forbid_control_characters);
802crate::counters::add(&crate::counters::STR_SCANS, 1);
803crate::counters::add(&crate::counters::STR_BYTES, (self.index - from) as u64);
804 }
805806fn skip_to_escape_impl(&mut self, forbid_control_characters: bool) {
807// Immediately bail-out on empty strings and consecutive escapes (e.g. \u041b\u0435)
808if self.index == self.slice.len()
809 || is_escape(self.slice[self.index], forbid_control_characters)
810 {
811return;
812 }
813self.index += 1;
814815let rest = &self.slice[self.index..];
816817if !forbid_control_characters {
818self.index += memchr::memchr2(b'"', b'\\', rest).unwrap_or(rest.len());
819return;
820 }
821822// We wish to find the first byte in range 0x00..=0x1F or " or \. Ideally, we'd use
823 // something akin to memchr3, but the memchr crate does not support this at the moment.
824 // Therefore, we use a variation on Mycroft's algorithm [1] to provide performance better
825 // than a naive loop. It runs faster than equivalent two-pass memchr2+SWAR code on
826 // benchmarks and it's cross-platform, so probably the right fit.
827 // [1]: https://groups.google.com/forum/#!original/comp.lang.c/2HtQXvg7iKc/xOJeipH6KLMJ
828829#[cfg(fast_arithmetic = "64")]
830type Chunk = u64;
831#[cfg(fast_arithmetic = "32")]
832type Chunk = u32;
833834const STEP: usize = mem::size_of::<Chunk>();
835const ONE_BYTES: Chunk = Chunk::MAX / 255; // 0x0101...01
836837for chunk in rest.chunks_exact(STEP) {
838let chars = Chunk::from_le_bytes(chunk.try_into().unwrap());
839let contains_ctrl = chars.wrapping_sub(ONE_BYTES * 0x20) & !chars;
840let chars_quote = chars ^ (ONE_BYTES * Chunk::from(b'"'));
841let contains_quote = chars_quote.wrapping_sub(ONE_BYTES) & !chars_quote;
842let chars_backslash = chars ^ (ONE_BYTES * Chunk::from(b'\\'));
843let contains_backslash = chars_backslash.wrapping_sub(ONE_BYTES) & !chars_backslash;
844let masked = (contains_ctrl | contains_quote | contains_backslash) & (ONE_BYTES << 7);
845if masked != 0 {
846// SAFETY: chunk is in-bounds for slice
847self.index = unsafe { chunk.as_ptr().offset_from(self.slice.as_ptr()) } as usize
848 + masked.trailing_zeros() as usize / 8;
849return;
850 }
851 }
852853self.index += rest.len() / STEP * STEP;
854self.skip_to_escape_slow();
855 }
856857#[cold]
858 #[inline(never)]
859fn skip_to_escape_slow(&mut self) {
860while self.index < self.slice.len() && !is_escape(self.slice[self.index], true) {
861self.index += 1;
862 }
863 }
864865/// The big optimization here over IoRead is that if the string contains no
866 /// backslash escape sequences, the returned &str is a slice of the raw JSON
867 /// data so we avoid copying into the scratch space.
868fn parse_str_bytes<'s, T, F>(
869&'s mut self,
870 scratch: &'s mut Vec<u8>,
871 validate: bool,
872 result: F,
873 ) -> Result<Reference<'a, 's, T>>
874where
875T: ?Sized + 's,
876 F: for<'f> FnOnce(&'s Self, &'f [u8]) -> Result<&'f T>,
877 {
878// Index of the first byte not yet copied into the scratch space.
879let mut start = self.index;
880881loop {
882self.skip_to_escape(validate);
883if self.index == self.slice.len() {
884return error(self, ErrorCode::EofWhileParsingString);
885 }
886match self.slice[self.index] {
887b'"' => {
888if scratch.is_empty() {
889// Fast path: return a slice of the raw JSON without any
890 // copying.
891crate::counters::add(&crate::counters::STR_BORROWS, 1);
892let borrowed = &self.slice[start..self.index];
893self.index += 1;
894return result(self, borrowed).map(Reference::Borrowed);
895 } else {
896crate::counters::add(&crate::counters::STR_COPIES, 1);
897scratch.extend_from_slice(&self.slice[start..self.index]);
898self.index += 1;
899return result(self, scratch).map(Reference::Copied);
900 }
901 }
902b'\\' => {
903scratch.extend_from_slice(&self.slice[start..self.index]);
904self.index += 1;
905match parse_escape(self, validate, scratch) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(parse_escape(self, validate, scratch));
906start = self.index;
907 }
908_ => {
909self.index += 1;
910return error(self, ErrorCode::ControlCharacterWhileParsingString);
911 }
912 }
913 }
914 }
915}
916917// ---------------------------------------------------------------------------
918// Whitespace scanning.
919//
920// Insignificant JSON whitespace is exactly ` `, `\n`, `\t`, `\r`. Every other
921// byte <= 0x20 is invalid where whitespace is allowed and must be RETURNED, not
922// skipped, so the parser reports it at the right line and column. That rules
923// out the tempting `b <= 0x20` test and any range test over 0x09..=0x0D, which
924// would silently accept the vertical tab and form feed.
925//
926// Why a wide scan is worth it here, measured rather than assumed: on
927// `citm_catalog.json` **98.0% of whitespace bytes live in runs of 8 or more**,
928// and 92.3% in runs of 17 to 32 -- deep indentation, one run per line. On
929// `twitter.json` it is 92.6% in runs of 8 or more. A per-byte loop walks all of
930// it one compare at a time. (The run *mean* alone would have said the opposite;
931// the distribution is what decides, and the two differ because most calls to
932// `skip_whitespace` find no whitespace at all.)
933// ---------------------------------------------------------------------------
934935// The SWAR primitives below serve the non-`accel` build only; with the island
936// on, the same exact predicates live in `rusty_json_turbo-accel`, tested there
937// against the same oracle.
938#[cfg(not(feature = "accel"))]
939const WS_ONES: u64 = 0x0101_0101_0101_0101;
940#[cfg(not(feature = "accel"))]
941const WS_HIGHS: u64 = 0x8080_8080_8080_8080;
942943#[inline(always)]
944fn is_ws(b: u8) -> bool {
945#[allow(non_exhaustive_omitted_patterns)] match b {
b' ' | b'\n' | b'\t' | b'\r' => true,
_ => false,
}matches!(b, b' ' | b'\n' | b'\t' | b'\r')946}
947948/// High bit set in each byte of `x` that is zero, and clear in every other.
949///
950/// Exact per byte, unlike the classic `(x - ONES) & !x & HIGHS`, whose
951/// subtraction borrows across byte boundaries and reports a byte as zero
952/// because its neighbour was. Here the addition is `low7 + 0x7F <= 0xFE`, which
953/// cannot carry out of its own byte, so no byte can be contaminated.
954#[cfg(not(feature = "accel"))]
955#[inline(always)]
956fn zero_bytes(x: u64) -> u64 {
957// `low7 + 0x7F` sets bit 7 iff low7 != 0; OR the original bit 7 back in, so
958 // the high bit marks a NON-zero byte. Invert for the zero bytes.
959!(((x & !WS_HIGHS).wrapping_add(!WS_HIGHS)) | x) & WS_HIGHS960}
961962/// High bit set in each byte of `x` equal to `c`.
963#[cfg(not(feature = "accel"))]
964#[inline(always)]
965fn eq_bytes(x: u64, c: u8) -> u64 {
966zero_bytes(x ^ (cas u64).wrapping_mul(WS_ONES))
967}
968969/// High bit set in each byte of `x` that is NOT insignificant whitespace.
970#[cfg(not(feature = "accel"))]
971#[inline(always)]
972fn non_ws_bytes(x: u64) -> u64 {
973let ws = eq_bytes(x, b' ') | eq_bytes(x, b'\n') | eq_bytes(x, b'\t') | eq_bytes(x, b'\r');
974 !ws & WS_HIGHS975}
976977// Both scanners return the byte they stopped on as well as its index.
978//
979// Returning only the index looked tidier and cost a measured 4% on
980// `canada.json`: the caller had to load that byte a second time, on every one
981// of the 557,593 calls a whitespace-free document makes. The file that CANNOT
982// benefit from a whitespace change is the one that exposed it -- which is the
983// whole reason to keep such a file in the corpus and to require that it reads
984// exactly the old number.
985986/// The oracle: the first byte at or after `from` that is not insignificant
987/// whitespace, with its index. One byte at a time.
988///
989/// Stays in the tree forever. It is what the wide scan is gated against, and
990/// what runs when the knob turns the wide path off.
991#[inline]
992// THE ORACLE STAYS, even when nothing calls it in a default build. With the
993// island linked, the wide path lives in `rusty_json_turbo-accel` and reaches
994// its OWN scalar twin through `RJT_ISA=scalar`, which leaves this one reachable
995// only from the `knobs` A/B and the tests below. It is kept regardless: a fast
996// path whose scalar twin has been deleted cannot be checked and cannot be
997// debugged, and this crate's tests compare against exactly this function.
998#[allow(dead_code)]
999fn scan_ws_scalar(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1000let mut i = from;
1001while i < slice.len() {
1002let b = slice[i];
1003if !is_ws(b) {
1004return (i, Some(b));
1005 }
1006 i += 1;
1007 }
1008 (i, None)
1009}
10101011/// How many bytes to walk one at a time before reaching for a wide step.
1012///
1013/// Not a guess. The run-length census says **46% of `twitter.json`'s whitespace
1014/// runs are a single byte** (13,345 of 28,826) and 34% of `citm_catalog`'s are.
1015/// Paying an eight-byte load and a SWAR test to discover that costs far more
1016/// than the two compares it replaces, and measured **0.903x on twitter scan,
1017/// 20/21** -- a real regression on a file with plenty of whitespace. Peeling a
1018/// few bytes first means a short run never reaches the wide path, while a long
1019/// one pays four extra compares out of seventeen or more.
1020const WS_PEEL: usize = 4;
10211022/// Skip a whitespace run that is already known to have started, and return the
1023/// byte that ended it.
1024///
1025/// Deliberately NOT `inline(always)`: it runs only when there is whitespace to
1026/// skip, so inlining it into every call site would bloat the hot path that
1027/// mostly finds no whitespace at all.
1028#[inline]
1029fn scan_ws_run(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1030#[cfg(feature = "knobs")]
1031if !ws_wide() {
1032return scan_ws_scalar(slice, from);
1033 }
10341035let mut i = from;
1036// Peel a short run without touching the wide path (see `WS_PEEL`).
1037 //
1038 // This still matters, and matters MORE the wider the step gets:
1039 // `twitter`'s whitespace run is a single byte 13,346 times, and an 8-byte
1040 // step over those cost 10% before this peel existed.
1041let peel = (from + WS_PEEL).min(slice.len());
1042while i < peel {
1043let b = slice[i];
1044if !is_ws(b) {
1045return (i, Some(b));
1046 }
1047 i += 1;
1048 }
1049scan_ws_wide(slice, i)
1050}
10511052/// The wide scan, handed to the SIMD island (M3).
1053///
1054/// `rusty_json_turbo-accel` runs the same predicate 16 or 32 bytes at a time
1055/// on x86-64, with the scalar walk as its permanent oracle and `RJT_ISA` to
1056/// choose a rung at runtime. It is a separate crate so that no `unsafe` for a
1057/// vector load is ever written in the parser.
1058#[cfg(feature = "accel")]
1059#[inline]
1060fn scan_ws_wide(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1061let at = rusty_json_turbo_accel::first_non_ws(slice, from);
1062// Return the byte with the index. Returning only the index cost a measured
1063 // 4% on `canada.json`, because the caller loaded it again.
1064(at, slice.get(at).copied())
1065}
10661067/// The wide scan without the island: 8 bytes per step, SWAR.
1068///
1069/// Written so there is no panicking branch at all: no indexing, no unwrap, no
1070/// unreachable. Both fallible steps are provably infallible here and fold
1071/// away, and if either ever were not, the scalar tail finishes the job
1072/// correctly rather than aborting.
1073#[cfg(not(feature = "accel"))]
1074fn scan_ws_wide(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1075let mut i = from;
1076while let Some(window) = slice.get(i..i + 8) {
1077let Ok(bytes) = <[u8; 8]>::try_from(window) else {
1078break;
1079 };
1080let chunk = u64::from_le_bytes(bytes);
1081let m = non_ws_bytes(chunk);
1082if m != 0 {
1083// Little-endian: byte 0 is the lowest address, so the first set
1084 // high bit marks the first non-whitespace byte. Take the byte out
1085 // of the word already in hand rather than loading it again.
1086let k = (m.trailing_zeros() >> 3) as usize;
1087return (i + k, Some((chunk >> (k * 8)) as u8));
1088 }
1089 i += 8;
1090 }
1091scan_ws_scalar(slice, i)
1092}
10931094// ---------------------------------------------------------------------------
1095// Digit runs.
1096// ---------------------------------------------------------------------------
10971098/// Are all eight bytes ASCII digits?
1099///
1100/// For a digit `c` in `0x30..=0x39`: `c & 0xF0 == 0x30`, and `c + 6` stays
1101/// inside `0x36..=0x3F` so `(c + 6) & 0xF0 == 0x30` too, giving `0x33` once the
1102/// second nibble is folded down. Any other byte fails its OWN test -- a
1103/// non-digit can carry into its neighbour, but it can never rescue itself, so
1104/// the chunk is always rejected and a false accept is impossible.
1105#[inline(always)]
1106fn all_8_digits(chunk: u64) -> bool {
1107const LO: u64 = 0x0606_0606_0606_0606;
1108const HI: u64 = 0xF0F0_F0F0_F0F0_F0F0;
1109const WANT: u64 = 0x3333_3333_3333_3333;
1110 ((chunk & HI) | (((chunk.wrapping_add(LO)) & HI) >> 4)) == WANT1111}
11121113/// The value of eight ASCII digits, first character most significant.
1114///
1115/// The standard three-multiply fold (as in fast_float and simdjson). `chunk`
1116/// must have come from a little-endian load, so the first character is the
1117/// lowest byte, and every byte must be a digit -- check [`all_8_digits`] first.
1118/// Every step is deliberately WRAPPING. The intermediate products overflow
1119/// `u64` by design and the answer is read out of the high half, so plain `*`
1120/// would be correct in a release build and **panic in a debug build** -- a
1121/// consumer-visible bug that only shows up off the happy path. The unit test
1122/// below runs in debug, which is how this was caught.
1123#[inline(always)]
1124fn eight_digits_value(chunk: u64) -> u32 {
1125const MASK: u64 = 0x0000_00FF_0000_00FF;
1126const MUL1: u64 = 0x000F_4240_0000_0064; // 100 + (1_000_000 << 32)
1127const MUL2: u64 = 0x0000_2710_0000_0001; // 1 + (10_000 << 32)
1128let v = chunk.wrapping_sub(0x3030_3030_3030_3030);
1129let v = v.wrapping_mul(10).wrapping_add(v >> 8);
1130let lo = (v & MASK).wrapping_mul(MUL1);
1131let hi = ((v >> 16) & MASK).wrapping_mul(MUL2);
1132 (lo.wrapping_add(hi) >> 32) as u321133}
11341135// A FOUR-digit step was built on top of this one, twice, and refuted twice.
1136// It is not here, and the reason is a law rather than an accident -- see
1137// `parse_decimal` in `de.rs` and brick B4 in `corpus/LEDGER.md`.
11381139/// Is the wide digit scan on? `knobs` builds only.
1140#[cfg(feature = "knobs")]
1141fn num_wide() -> bool {
1142static V: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1143*V.get_or_init(|| std::env::var("RJT_NUM_WIDE").map_or(true, |v| v != "0"))
1144}
11451146/// Is the wide whitespace scan on? `knobs` builds only.
1147#[cfg(feature = "knobs")]
1148fn ws_wide() -> bool {
1149static V: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1150*V.get_or_init(|| std::env::var("RJT_WS_WIDE").map_or(true, |v| v != "0"))
1151}
11521153/// The first non-whitespace byte at or after `from`, with its index.
1154///
1155/// `inline(always)`, and tiny on purpose. The overwhelmingly common case is
1156/// that there is no whitespace to skip at all -- `canada.json` takes it on
1157/// essentially all of its 557,593 calls, `citm_catalog` on 92,950 of 169,287 --
1158/// and that case must cost exactly what the original `peek()` cost: one load,
1159/// one test. Everything else is behind a call.
1160#[inline(always)]
1161fn scan_ws(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1162match slice.get(from) {
1163Some(&b) if !is_ws(b) => (from, Some(b)),
1164None => (from, None),
1165_ => scan_ws_run(slice, from),
1166 }
1167}
11681169/// Is the whitespace fast path on? `knobs` builds only; cached, so the
1170/// environment is read once per process rather than once per whitespace run.
1171///
1172/// This is the A/B instrument for the whitespace brick: one binary, one code
1173/// layout, one env var between the arms. Comparing two *builds* instead would
1174/// confound the change with its layout, and layout alone is worth +-13% here.
1175#[cfg(feature = "knobs")]
1176fn ws_fastpath() -> bool {
1177static V: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1178*V.get_or_init(|| std::env::var("RJT_WS_FASTPATH").map_or(true, |v| v != "0"))
1179}
11801181impl<'a> private::Sealedfor SliceRead<'a> {}
11821183impl<'a> Read<'a> for SliceRead<'a> {
1184#[inline]
1185fn next(&mut self) -> Result<Option<u8>> {
1186// `Ok(self.slice.get(self.index).map(|ch| { self.index += 1; *ch }))`
1187 // is about 10% slower.
1188crate::counters::add(&crate::counters::NEXT, 1);
1189Ok(if self.index < self.slice.len() {
1190let ch = self.slice[self.index];
1191self.index += 1;
1192Some(ch)
1193 } else {
1194None1195 })
1196 }
11971198#[inline]
1199fn peek(&mut self) -> Result<Option<u8>> {
1200// `Ok(self.slice.get(self.index).map(|ch| *ch))` is about 10% slower
1201 // for some reason.
1202crate::counters::add(&crate::counters::PEEK, 1);
1203Ok(if self.index < self.slice.len() {
1204Some(self.slice[self.index])
1205 } else {
1206None1207 })
1208 }
12091210#[inline]
1211fn discard(&mut self) {
1212crate::counters::add(&crate::counters::DISCARD, 1);
1213self.index += 1;
1214 }
12151216#[inline]
1217fn take_8_digits(&mut self) -> Option<u32> {
1218#[cfg(feature = "knobs")]
1219if !num_wide() {
1220return None;
1221 }
1222// No `?`: the crate denies `clippy::question_mark_used`.
1223let Some(window) = self.slice.get(self.index..self.index + 8) else {
1224return None;
1225 };
1226let Ok(bytes) = <[u8; 8]>::try_from(window) else {
1227return None;
1228 };
1229let chunk = u64::from_le_bytes(bytes);
1230crate::counters::add(&crate::counters::D8_CALLS, 1);
1231if all_8_digits(chunk) {
1232crate::counters::add(&crate::counters::D8_HITS, 1);
1233self.index += 8;
1234Some(eight_digits_value(chunk))
1235 } else {
1236None1237 }
1238 }
12391240#[inline]
1241fn skip_whitespace(&mut self) -> Result<Option<u8>> {
1242// MEASUREMENT ARM, `profile` builds only: `RJT_WS_FASTPATH=0` forces the
1243 // old byte-at-a-time route so both arms can be COUNTED in one binary.
1244 // The knob is resolved once and cached, and it is read once per
1245 // whitespace run rather than per byte, so it does not itself become the
1246 // thing being measured.
1247#[cfg(feature = "knobs")]
1248if !ws_fastpath() {
1249loop {
1250match tri!(self.peek()) {
1251Some(b' ' | b'\n' | b'\t' | b'\r') => self.discard(),
1252 other => return Ok(other),
1253 }
1254 }
1255 }
12561257// One walk of the slice, eight bytes at a time where the run is long
1258 // enough, instead of a `Result<Option<u8>>` round trip per byte through
1259 // peek()/discard(). Byte-for-byte the same decision as the default
1260 // loop: exactly the four insignificant whitespace bytes are skipped,
1261 // and the first byte that is not one of them is returned WITHOUT being
1262 // consumed.
1263let slice = self.slice;
1264let start = self.index;
1265let (i, next) = scan_ws(slice, start);
1266self.index = i;
1267crate::counters::add(&crate::counters::WS_RUNS, 1);
1268crate::counters::add(&crate::counters::WS_BYTES, (i - start) as u64);
1269Ok(next)
1270 }
12711272fn position(&self) -> Position {
1273self.position_of_index(self.index)
1274 }
12751276fn peek_position(&self) -> Position {
1277// Cap it at slice.len() just in case the most recent call was next()
1278 // and it returned the last byte.
1279self.position_of_index(cmp::min(self.slice.len(), self.index + 1))
1280 }
12811282fn byte_offset(&self) -> usize {
1283self.index
1284 }
12851286fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'a, 's, str>> {
1287self.parse_str_bytes(scratch, true, as_str)
1288 }
12891290fn parse_str_raw<'s>(
1291&'s mut self,
1292 scratch: &'s mut Vec<u8>,
1293 ) -> Result<Reference<'a, 's, [u8]>> {
1294self.parse_str_bytes(scratch, false, |_, bytes| Ok(bytes))
1295 }
12961297fn ignore_str(&mut self) -> Result<()> {
1298loop {
1299self.skip_to_escape(true);
1300if self.index == self.slice.len() {
1301return error(self, ErrorCode::EofWhileParsingString);
1302 }
1303match self.slice[self.index] {
1304b'"' => {
1305self.index += 1;
1306return Ok(());
1307 }
1308b'\\' => {
1309self.index += 1;
1310match ignore_escape(self) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(ignore_escape(self));
1311 }
1312_ => {
1313return error(self, ErrorCode::ControlCharacterWhileParsingString);
1314 }
1315 }
1316 }
1317 }
13181319#[inline]
1320fn decode_hex_escape(&mut self) -> Result<u16> {
1321match self.slice[self.index..] {
1322 [a, b, c, d, ..] => {
1323self.index += 4;
1324match decode_four_hex_digits(a, b, c, d) {
1325Some(val) => Ok(val),
1326None => error(self, ErrorCode::InvalidEscape),
1327 }
1328 }
1329_ => {
1330self.index = self.slice.len();
1331error(self, ErrorCode::EofWhileParsingString)
1332 }
1333 }
1334 }
13351336#[cfg(feature = "raw_value")]
1337fn begin_raw_buffering(&mut self) {
1338self.raw_buffering_start_index = self.index;
1339 }
13401341#[cfg(feature = "raw_value")]
1342fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
1343where
1344V: Visitor<'a>,
1345 {
1346let raw = &self.slice[self.raw_buffering_start_index..self.index];
1347let raw = match str::from_utf8(raw) {
1348Ok(raw) => raw,
1349Err(_) => return error(self, ErrorCode::InvalidUnicodeCodePoint),
1350 };
1351visitor.visit_map(BorrowedRawDeserializer {
1352 raw_value: Some(raw),
1353 })
1354 }
13551356const should_early_return_if_failed: bool = false;
13571358#[inline]
1359 #[cold]
1360fn set_failed(&mut self, _failed: &mut bool) {
1361self.slice = &self.slice[..self.index];
1362 }
1363}
13641365//////////////////////////////////////////////////////////////////////////////
13661367impl<'a> StrRead<'a> {
1368/// Create a JSON input source to read from a UTF-8 string.
1369pub fn new(s: &'a str) -> Self {
1370StrRead {
1371 delegate: SliceRead::new(s.as_bytes()),
1372#[cfg(feature = "raw_value")]
1373data: s,
1374 }
1375 }
1376}
13771378impl<'a> private::Sealedfor StrRead<'a> {}
13791380impl<'a> Read<'a> for StrRead<'a> {
1381#[inline]
1382fn next(&mut self) -> Result<Option<u8>> {
1383self.delegate.next()
1384 }
13851386#[inline]
1387fn peek(&mut self) -> Result<Option<u8>> {
1388self.delegate.peek()
1389 }
13901391#[inline]
1392fn discard(&mut self) {
1393self.delegate.discard();
1394 }
13951396#[inline]
1397fn skip_whitespace(&mut self) -> Result<Option<u8>> {
1398self.delegate.skip_whitespace()
1399 }
14001401#[inline]
1402fn take_8_digits(&mut self) -> Option<u32> {
1403self.delegate.take_8_digits()
1404 }
14051406#[inline]
1407fn position(&self) -> Position {
1408self.delegate.position()
1409 }
14101411fn peek_position(&self) -> Position {
1412self.delegate.peek_position()
1413 }
14141415fn byte_offset(&self) -> usize {
1416self.delegate.byte_offset()
1417 }
14181419fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'a, 's, str>> {
1420self.delegate.parse_str_bytes(scratch, true, |_, bytes| {
1421// The deserialization input came in as &str with a UTF-8 guarantee,
1422 // and the \u-escapes are checked along the way, so don't need to
1423 // check here.
1424Ok(unsafe { str::from_utf8_unchecked(bytes) })
1425 })
1426 }
14271428fn parse_str_raw<'s>(
1429&'s mut self,
1430 scratch: &'s mut Vec<u8>,
1431 ) -> Result<Reference<'a, 's, [u8]>> {
1432self.delegate.parse_str_raw(scratch)
1433 }
14341435fn ignore_str(&mut self) -> Result<()> {
1436self.delegate.ignore_str()
1437 }
14381439fn decode_hex_escape(&mut self) -> Result<u16> {
1440self.delegate.decode_hex_escape()
1441 }
14421443#[cfg(feature = "raw_value")]
1444fn begin_raw_buffering(&mut self) {
1445self.delegate.begin_raw_buffering();
1446 }
14471448#[cfg(feature = "raw_value")]
1449fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
1450where
1451V: Visitor<'a>,
1452 {
1453let raw = &self.data[self.delegate.raw_buffering_start_index..self.delegate.index];
1454visitor.visit_map(BorrowedRawDeserializer {
1455 raw_value: Some(raw),
1456 })
1457 }
14581459const should_early_return_if_failed: bool = false;
14601461#[inline]
1462 #[cold]
1463fn set_failed(&mut self, failed: &mut bool) {
1464self.delegate.set_failed(failed);
1465 }
1466}
14671468//////////////////////////////////////////////////////////////////////////////
14691470impl<'de, R> private::Sealedfor &mut R where R: Read<'de> {}
14711472impl<'de, R> Read<'de> for &mut R
1473where
1474R: Read<'de>,
1475{
1476fn next(&mut self) -> Result<Option<u8>> {
1477 R::next(self)
1478 }
14791480fn peek(&mut self) -> Result<Option<u8>> {
1481 R::peek(self)
1482 }
14831484fn discard(&mut self) {
1485 R::discard(self);
1486 }
14871488fn position(&self) -> Position {
1489 R::position(self)
1490 }
14911492fn peek_position(&self) -> Position {
1493 R::peek_position(self)
1494 }
14951496fn byte_offset(&self) -> usize {
1497 R::byte_offset(self)
1498 }
14991500fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'de, 's, str>> {
1501 R::parse_str(self, scratch)
1502 }
15031504fn parse_str_raw<'s>(
1505&'s mut self,
1506 scratch: &'s mut Vec<u8>,
1507 ) -> Result<Reference<'de, 's, [u8]>> {
1508 R::parse_str_raw(self, scratch)
1509 }
15101511fn ignore_str(&mut self) -> Result<()> {
1512 R::ignore_str(self)
1513 }
15141515fn decode_hex_escape(&mut self) -> Result<u16> {
1516 R::decode_hex_escape(self)
1517 }
15181519#[cfg(feature = "raw_value")]
1520fn begin_raw_buffering(&mut self) {
1521 R::begin_raw_buffering(self);
1522 }
15231524#[cfg(feature = "raw_value")]
1525fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
1526where
1527V: Visitor<'de>,
1528 {
1529 R::end_raw_buffering(self, visitor)
1530 }
15311532const should_early_return_if_failed: bool = R::should_early_return_if_failed;
15331534fn set_failed(&mut self, failed: &mut bool) {
1535 R::set_failed(self, failed);
1536 }
1537}
15381539//////////////////////////////////////////////////////////////////////////////
15401541/// Marker for whether StreamDeserializer can implement FusedIterator.
1542pub trait Fused: private::Sealed {}
1543impl<'a> Fusedfor SliceRead<'a> {}
1544impl<'a> Fusedfor StrRead<'a> {}
15451546fn is_escape(ch: u8, including_control_characters: bool) -> bool {
1547ch == b'"' || ch == b'\\' || (including_control_characters && ch < 0x20)
1548}
15491550fn next_or_eof<'de, R>(read: &mut R) -> Result<u8>
1551where
1552R: ?Sized + Read<'de>,
1553{
1554match match read.next() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.next()) {
1555Some(b) => Ok(b),
1556None => error(read, ErrorCode::EofWhileParsingString),
1557 }
1558}
15591560fn peek_or_eof<'de, R>(read: &mut R) -> Result<u8>
1561where
1562R: ?Sized + Read<'de>,
1563{
1564match match read.peek() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.peek()) {
1565Some(b) => Ok(b),
1566None => error(read, ErrorCode::EofWhileParsingString),
1567 }
1568}
15691570fn error<'de, R, T>(read: &R, reason: ErrorCode) -> Result<T>
1571where
1572R: ?Sized + Read<'de>,
1573{
1574let position = read.position();
1575Err(Error::syntax(reason, position.line, position.column))
1576}
15771578fn as_str<'de, 's, R: Read<'de>>(read: &R, slice: &'s [u8]) -> Result<&'s str> {
1579// PROBE (brick B12): this is the whole UTF-8 validation bill for
1580 // `from_slice`. `from_str` pays none of it, so the two together bound the
1581 // brick without any new code needing to be written.
1582crate::counters::add(&crate::counters::UTF8_CALLS, 1);
1583crate::counters::add(&crate::counters::UTF8_BYTES, slice.len() as u64);
1584 str::from_utf8(slice).or_else(|_| error(read, ErrorCode::InvalidUnicodeCodePoint))
1585}
15861587/// Parses a JSON escape sequence and appends it into the scratch space. Assumes
1588/// the previous byte read was a backslash.
1589fn parse_escape<'de, R: Read<'de>>(
1590 read: &mut R,
1591 validate: bool,
1592 scratch: &mut Vec<u8>,
1593) -> Result<()> {
1594let ch = match next_or_eof(read) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(read));
15951596match ch {
1597b'"' => scratch.push(b'"'),
1598b'\\' => scratch.push(b'\\'),
1599b'/' => scratch.push(b'/'),
1600b'b' => scratch.push(b'\x08'),
1601b'f' => scratch.push(b'\x0c'),
1602b'n' => scratch.push(b'\n'),
1603b'r' => scratch.push(b'\r'),
1604b't' => scratch.push(b'\t'),
1605b'u' => return parse_unicode_escape(read, validate, scratch),
1606_ => return error(read, ErrorCode::InvalidEscape),
1607 }
16081609Ok(())
1610}
16111612/// Parses a JSON \u escape and appends it into the scratch space. Assumes `\u`
1613/// has just been read.
1614#[cold]
1615fn parse_unicode_escape<'de, R: Read<'de>>(
1616 read: &mut R,
1617 validate: bool,
1618 scratch: &mut Vec<u8>,
1619) -> Result<()> {
1620let mut n = match read.decode_hex_escape() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.decode_hex_escape());
16211622// Non-BMP characters are encoded as a sequence of two hex escapes,
1623 // representing UTF-16 surrogates. If deserializing a utf-8 string the
1624 // surrogates are required to be paired, whereas deserializing a byte string
1625 // accepts lone surrogates.
1626if validate && n >= 0xDC00 && n <= 0xDFFF {
1627// XXX: This is actually a trailing surrogate.
1628return error(read, ErrorCode::LoneLeadingSurrogateInHexEscape);
1629 }
16301631loop {
1632if n < 0xD800 || n > 0xDBFF {
1633// Every u16 outside of the surrogate ranges is guaranteed to be a
1634 // legal char.
1635push_wtf8_codepoint(nas u32, scratch);
1636return Ok(());
1637 }
16381639// n is a leading surrogate, we now expect a trailing surrogate.
1640let n1 = n;
16411642if match peek_or_eof(read) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(peek_or_eof(read)) == b'\\' {
1643read.discard();
1644 } else {
1645return if validate {
1646read.discard();
1647error(read, ErrorCode::UnexpectedEndOfHexEscape)
1648 } else {
1649push_wtf8_codepoint(n1as u32, scratch);
1650Ok(())
1651 };
1652 }
16531654if match peek_or_eof(read) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(peek_or_eof(read)) == b'u' {
1655read.discard();
1656 } else {
1657return if validate {
1658read.discard();
1659error(read, ErrorCode::UnexpectedEndOfHexEscape)
1660 } else {
1661push_wtf8_codepoint(n1as u32, scratch);
1662// The \ prior to this byte started an escape sequence, so we
1663 // need to parse that now. This recursive call does not blow the
1664 // stack on malicious input because the escape is not \u, so it
1665 // will be handled by one of the easy nonrecursive cases.
1666parse_escape(read, validate, scratch)
1667 };
1668 }
16691670let n2 = match read.decode_hex_escape() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.decode_hex_escape());
16711672if n2 < 0xDC00 || n2 > 0xDFFF {
1673if validate {
1674return error(read, ErrorCode::LoneLeadingSurrogateInHexEscape);
1675 }
1676push_wtf8_codepoint(n1as u32, scratch);
1677// If n2 is a leading surrogate, we need to restart.
1678n = n2;
1679continue;
1680 }
16811682// This value is in range U+10000..=U+10FFFF, which is always a valid
1683 // codepoint.
1684let n = ((((n1 - 0xD800) as u32) << 10) | (n2 - 0xDC00) as u32) + 0x1_0000;
1685push_wtf8_codepoint(n, scratch);
1686return Ok(());
1687 }
1688}
16891690/// Adds a WTF-8 codepoint to the end of the buffer. This is a more efficient
1691/// implementation of String::push. The codepoint may be a surrogate.
1692#[inline]
1693fn push_wtf8_codepoint(n: u32, scratch: &mut Vec<u8>) {
1694if n < 0x80 {
1695scratch.push(nas u8);
1696return;
1697 }
16981699scratch.reserve(4);
17001701// SAFETY: After the `reserve` call, `scratch` has at least 4 bytes of
1702 // allocated but uninitialized memory after its last initialized byte, which
1703 // is where `ptr` points. All reachable match arms write `encoded_len` bytes
1704 // to that region and update the length accordingly, and `encoded_len` is
1705 // always <= 4.
1706unsafe {
1707let ptr = scratch.as_mut_ptr().add(scratch.len());
17081709let encoded_len = match n {
17100..=0x7F => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
17110x80..=0x7FF => {
1712ptr.write(((n >> 6) & 0b0001_1111) as u8 | 0b1100_0000);
17132
1714}
17150x800..=0xFFFF => {
1716ptr.write(((n >> 12) & 0b0000_1111) as u8 | 0b1110_0000);
1717ptr.add(1)
1718 .write(((n >> 6) & 0b0011_1111) as u8 | 0b1000_0000);
17193
1720}
17210x1_0000..=0x10_FFFF => {
1722ptr.write(((n >> 18) & 0b0000_0111) as u8 | 0b1111_0000);
1723ptr.add(1)
1724 .write(((n >> 12) & 0b0011_1111) as u8 | 0b1000_0000);
1725ptr.add(2)
1726 .write(((n >> 6) & 0b0011_1111) as u8 | 0b1000_0000);
17274
1728}
17290x11_0000.. => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1730 };
1731ptr.add(encoded_len - 1)
1732 .write((n & 0b0011_1111) as u8 | 0b1000_0000);
17331734scratch.set_len(scratch.len() + encoded_len);
1735 }
1736}
17371738/// Parses a JSON escape sequence and discards the value. Assumes the previous
1739/// byte read was a backslash.
1740fn ignore_escape<'de, R>(read: &mut R) -> Result<()>
1741where
1742R: ?Sized + Read<'de>,
1743{
1744let ch = match next_or_eof(read) {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(read));
17451746match ch {
1747b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't' => {}
1748b'u' => {
1749// At this point we don't care if the codepoint is valid. We just
1750 // want to consume it. We don't actually know what is valid or not
1751 // at this point, because that depends on if this string will
1752 // ultimately be parsed into a string or a byte buffer in the "real"
1753 // parse.
17541755match read.decode_hex_escape() {
core::result::Result::Ok(val) => val,
core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(read.decode_hex_escape());
1756 }
1757_ => {
1758return error(read, ErrorCode::InvalidEscape);
1759 }
1760 }
17611762Ok(())
1763}
17641765const fn decode_hex_val_slow(val: u8) -> Option<u8> {
1766match val {
1767b'0'..=b'9' => Some(val - b'0'),
1768b'A'..=b'F' => Some(val - b'A' + 10),
1769b'a'..=b'f' => Some(val - b'a' + 10),
1770_ => None,
1771 }
1772}
17731774const fn build_hex_table(shift: usize) -> [i16; 256] {
1775let mut table = [0; 256];
1776let mut ch = 0;
1777while ch < 256 {
1778 table[ch] = match decode_hex_val_slow(ch as u8) {
1779Some(val) => (val as i16) << shift,
1780None => -1,
1781 };
1782 ch += 1;
1783 }
1784table1785}
17861787static HEX0: [i16; 256] = build_hex_table(0);
1788static HEX1: [i16; 256] = build_hex_table(4);
17891790fn decode_four_hex_digits(a: u8, b: u8, c: u8, d: u8) -> Option<u16> {
1791let a = HEX1[aas usize] as i32;
1792let b = HEX0[bas usize] as i32;
1793let c = HEX1[cas usize] as i32;
1794let d = HEX0[das usize] as i32;
17951796let codepoint = ((a | b) << 8) | c | d;
17971798// A single sign bit check.
1799if codepoint >= 0 {
1800Some(codepointas u16)
1801 } else {
1802None1803 }
1804}
18051806#[cfg(test)]
1807mod ws_tests {
1808use super::{is_ws, scan_ws, scan_ws_scalar};
18091810/// Index only, for the comparisons below.
1811fn wide(s: &[u8], from: usize) -> usize {
1812 scan_ws(s, from).0
1813}
1814fn scalar(s: &[u8], from: usize) -> usize {
1815 scan_ws_scalar(s, from).0
1816}
18171818/// The byte each scanner reports must be the byte at the index it reports.
1819 /// Returning a stale or re-loaded byte here would be invisible to an
1820 /// index-only comparison and would corrupt every token boundary.
1821fn agree(s: &[u8], from: usize) {
1822let (wi, wb) = scan_ws(s, from);
1823let (si, sb) = scan_ws_scalar(s, from);
1824assert_eq!(wi, si, "index disagreement at {from} in {s:?}");
1825assert_eq!(wb, sb, "byte disagreement at {from} in {s:?}");
1826assert_eq!(wb, s.get(wi).copied(), "byte is not the one at the index");
1827 }
1828use alloc::vec;
1829use alloc::vec::Vec;
18301831/// The gate the house requires of any wide kernel: it must agree with its
1832 /// scalar twin on every byte value at every offset, not merely on a corpus.
1833#[test]
1834fn wide_matches_scalar_for_every_byte_at_every_offset() {
1835// A window long enough that the wide loop runs at least twice, so a
1836 // byte can be placed before, inside and after the first chunk.
1837for pos in 0..24usize {
1838for b in 0..=255u8 {
1839let mut buf = [b' '; 24];
1840 buf[pos] = b;
1841for from in 0..24usize {
1842let want = scalar(&buf, from);
1843let got = wide(&buf, from);
1844assert_eq!(
1845 got, want,
1846"byte {b:#04x} at {pos}, scanning from {from}: wide said {got}, scalar {want}"
1847);
1848// ... and the byte reported must be the byte at that index.
1849agree(&buf, from);
1850 }
1851 }
1852 }
1853 }
18541855/// Every length from empty to past two chunks, all-whitespace and none.
1856#[test]
1857fn wide_matches_scalar_at_every_length() {
1858for len in 0..40usize {
1859for fill in [b' ', b'\t', b'\n', b'\r', b'x', 0x0b, 0x0c, 0x00] {
1860let buf = vec![fill; len];
1861for from in 0..=len {
1862assert_eq!(
1863 wide(&buf, from),
1864 scalar(&buf, from),
1865"fill {fill:#04x}, len {len}, from {from}"
1866);
1867 agree(&buf, from);
1868 }
1869 }
1870 }
1871 }
18721873/// Mixed content, including the bytes a range test would wrongly skip.
1874#[test]
1875fn wide_matches_scalar_on_mixed_content() {
1876let mut seed = 0x1234_5678_9abc_def0u64;
1877let alphabet = [
1878b' ', b' ', b' ', b'\t', b'\n', b'\r', // whitespace, over-represented
18790x0b, 0x0c, 0x00, 0x1f, // NOT whitespace, and <= 0x20: the trap
1880b'{', b'}', b'"', b'1', b'a', 0xff,
1881 ];
1882for _ in 0..2000 {
1883let mut buf = Vec::new();
1884for _ in 0..64 {
1885 seed = seed
1886 .wrapping_mul(6364136223846793005)
1887 .wrapping_add(1442695040888963407);
1888 buf.push(alphabet[(seed >> 33) as usize % alphabet.len()]);
1889 }
1890for from in 0..buf.len() {
1891assert_eq!(
1892 wide(&buf, from),
1893 scalar(&buf, from),
1894"buf {buf:?} from {from}"
1895);
1896 agree(&buf, from);
1897 }
1898 }
1899 }
19001901/// The control chars a `b <= 0x20` or `0x09..=0x0d` test would wrongly
1902 /// treat as whitespace. Skipping them would silently accept invalid JSON.
1903#[test]
1904fn vertical_tab_and_form_feed_are_not_whitespace() {
1905for b in [0x0b_u8, 0x0c, 0x00, 0x01, 0x1f, 0x21] {
1906assert!(!is_ws(b), "{b:#04x} must not be skippable whitespace");
1907let mut buf = [b' '; 16];
1908 buf[10] = b;
1909assert_eq!(wide(&buf, 0), 10);
1910assert_eq!(scalar(&buf, 0), 10);
1911 }
1912 }
1913}
19141915#[cfg(test)]
1916mod digit_tests {
1917use super::{all_8_digits, eight_digits_value};
1918use alloc::format;
19191920/// What the SWAR fold must agree with, computed the obvious way.
1921fn scalar_value(s: &[u8; 8]) -> u32 {
1922 s.iter()
1923 .fold(0u32, |acc, &c| acc * 10 + u32::from(c - b'0'))
1924 }
19251926#[test]
1927fn accepts_only_all_digit_chunks() {
1928let digits = *b"12345678";
1929assert!(all_8_digits(u64::from_le_bytes(digits)));
1930// A single non-digit anywhere must reject the chunk. Byte values are
1931 // exhaustive, positions are exhaustive.
1932for pos in 0..8 {
1933for b in 0..=255u8 {
1934let mut buf = digits;
1935 buf[pos] = b;
1936let want = b.is_ascii_digit();
1937assert_eq!(
1938 all_8_digits(u64::from_le_bytes(buf)),
1939 want,
1940"byte {b:#04x} at {pos}"
1941);
1942 }
1943 }
1944 }
19451946#[test]
1947fn value_matches_scalar_on_every_boundary_and_at_random() {
1948for s in [
1949b"00000000",
1950b"00000001",
1951b"10000000",
1952b"99999999",
1953b"12345678",
1954b"87654321",
1955b"09090909",
1956b"90909090",
1957 ] {
1958let chunk = u64::from_le_bytes(*s);
1959assert!(all_8_digits(chunk));
1960assert_eq!(eight_digits_value(chunk), scalar_value(s), "{:?}", s);
1961 }
1962// Every digit in every position, the rest zeroes: catches a fold that
1963 // weights a position wrongly.
1964for pos in 0..8 {
1965for d in b'0'..=b'9' {
1966let mut buf = *b"00000000";
1967 buf[pos] = d;
1968let chunk = u64::from_le_bytes(buf);
1969assert_eq!(
1970 eight_digits_value(chunk),
1971 scalar_value(&buf),
1972"digit {} at {pos}",
1973 d as char
1974 );
1975 }
1976 }
1977let mut seed = 0x9e37_79b9_7f4a_7c15u64;
1978for _ in 0..20_000 {
1979 seed = seed
1980 .wrapping_mul(6364136223846793005)
1981 .wrapping_add(1442695040888963407);
1982let n = (seed >> 32) % 100_000_000;
1983let s = format!("{n:08}");
1984let buf = <[u8; 8]>::try_from(s.as_bytes()).unwrap();
1985let chunk = u64::from_le_bytes(buf);
1986assert!(all_8_digits(chunk), "{s}");
1987assert_eq!(eight_digits_value(chunk), n as u32, "{s}");
1988 }
1989 }
1990}