Skip to main content

serde_json/
read.rs

1use crate::error::{Error, ErrorCode, Result};
2use alloc::vec::Vec;
3use core::cmp;
4use core::mem;
5use core::ops::Deref;
6use core::str;
7
8#[cfg(feature = "std")]
9use crate::io;
10
11#[cfg(feature = "raw_value")]
12use crate::raw::BorrowedRawDeserializer;
13#[cfg(all(feature = "raw_value", feature = "std"))]
14use crate::raw::OwnedRawDeserializer;
15#[cfg(all(feature = "raw_value", feature = "std"))]
16use alloc::string::String;
17#[cfg(feature = "raw_value")]
18use serde::de::Visitor;
19
20/// Trait used by the deserializer for iterating over input. This is manually
21/// "specialized" for iterating over `&[u8]`. Once feature(specialization) is
22/// stable we can use actual specialization.
23///
24/// This trait is sealed and cannot be implemented for types outside of
25/// `serde_json`.
26pub trait Read<'de>: private::Sealed {
27    #[doc(hidden)]
28    fn next(&mut self) -> Result<Option<u8>>;
29    #[doc(hidden)]
30    fn peek(&mut self) -> Result<Option<u8>>;
31
32    /// Only valid after a call to peek(). Discards the peeked byte.
33    #[doc(hidden)]
34    fn discard(&mut self);
35
36    /// Skip insignificant whitespace and peek the next byte, without consuming
37    /// it. Equivalent to `while peek() is whitespace { discard() }; peek()`.
38    ///
39    /// This exists as its own method because whitespace is the single largest
40    /// class of byte in pretty-printed JSON -- 71.9% of `citm_catalog.json`,
41    /// where skipping it measured **36% of the time to parse the document into
42    /// a `Value`**. Going through `peek()`/`discard()` costs a `Result<Option>`
43    /// construction, a match and a bounds check for every one of those bytes.
44    /// A reader that owns a contiguous buffer can do far better, so it gets to
45    /// override this; the default is exactly the loop it replaces.
46    ///
47    /// Only ` `, `\n`, `\t` and `\r` are insignificant. Any other byte <= 0x20
48    /// is invalid at this position and must be RETURNED rather than skipped, so
49    /// that the parser reports it at the right line and column.
50    #[doc(hidden)]
51    #[inline]
52    fn skip_whitespace(&mut self) -> Result<Option<u8>> {
53        loop {
54            match match self.peek() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(self.peek()) {
55                Some(b' ' | b'\n' | b'\t' | b'\r') => self.discard(),
56                other => return Ok(other),
57            }
58        }
59    }
60
61    /// If the next eight bytes are all ASCII digits, consume them and return
62    /// their value; otherwise consume nothing and return `None`.
63    ///
64    /// The number path is the whitespace path all over again: `canada.json` is
65    /// **90.1% number bytes** -- 2.03 MB of digits across 111,126 numbers,
66    /// about 17 digits each -- and every one of them goes through a
67    /// `peek_or_null()`, a range test, an overflow check and an `eat_char()`.
68    /// After the whitespace work that file still made 2.19 million `peek` and
69    /// 2.14 million `discard` calls, essentially all of them here.
70    ///
71    /// The caller must only use this while an overflow is impossible for the
72    /// whole chunk, so that the per-digit overflow check it replaces could not
73    /// have fired either. The default implementation declines, which is correct
74    /// for any reader that is not backed by a contiguous buffer.
75    #[doc(hidden)]
76    #[inline]
77    fn take_8_digits(&mut self) -> Option<u32> {
78        None
79    }
80
81    /// Position of the most recent call to next().
82    ///
83    /// The most recent call was probably next() and not peek(), but this method
84    /// should try to return a sensible result if the most recent call was
85    /// actually peek() because we don't always know.
86    ///
87    /// Only called in case of an error, so performance is not important.
88    #[doc(hidden)]
89    fn position(&self) -> Position;
90
91    /// Position of the most recent call to peek().
92    ///
93    /// The most recent call was probably peek() and not next(), but this method
94    /// should try to return a sensible result if the most recent call was
95    /// actually next() because we don't always know.
96    ///
97    /// Only called in case of an error, so performance is not important.
98    #[doc(hidden)]
99    fn peek_position(&self) -> Position;
100
101    /// Offset from the beginning of the input to the next byte that would be
102    /// returned by next() or peek().
103    #[doc(hidden)]
104    fn byte_offset(&self) -> usize;
105
106    /// Assumes the previous byte was a quotation mark. Parses a JSON-escaped
107    /// string until the next quotation mark using the given scratch space if
108    /// necessary. The scratch space is initially empty.
109    #[doc(hidden)]
110    fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'de, 's, str>>;
111
112    /// Assumes the previous byte was a quotation mark. Parses a JSON-escaped
113    /// string until the next quotation mark using the given scratch space if
114    /// necessary. The scratch space is initially empty.
115    ///
116    /// This function returns the raw bytes in the string with escape sequences
117    /// expanded but without performing unicode validation.
118    #[doc(hidden)]
119    fn parse_str_raw<'s>(
120        &'s mut self,
121        scratch: &'s mut Vec<u8>,
122    ) -> Result<Reference<'de, 's, [u8]>>;
123
124    /// Assumes the previous byte was a quotation mark. Parses a JSON-escaped
125    /// string until the next quotation mark but discards the data.
126    #[doc(hidden)]
127    fn ignore_str(&mut self) -> Result<()>;
128
129    /// Assumes the previous byte was a hex escape sequence ('\u') in a string.
130    /// Parses next hexadecimal sequence.
131    #[doc(hidden)]
132    fn decode_hex_escape(&mut self) -> Result<u16>;
133
134    /// Switch raw buffering mode on.
135    ///
136    /// This is used when deserializing `RawValue`.
137    #[cfg(feature = "raw_value")]
138    #[doc(hidden)]
139    fn begin_raw_buffering(&mut self);
140
141    /// Switch raw buffering mode off and provides the raw buffered data to the
142    /// given visitor.
143    #[cfg(feature = "raw_value")]
144    #[doc(hidden)]
145    fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
146    where
147        V: Visitor<'de>;
148
149    /// Whether StreamDeserializer::next needs to check the failed flag. True
150    /// for IoRead, false for StrRead and SliceRead which can track failure by
151    /// truncating their input slice to avoid the extra check on every next
152    /// call.
153    #[doc(hidden)]
154    const should_early_return_if_failed: bool;
155
156    /// Mark a persistent failure of StreamDeserializer, either by setting the
157    /// flag or by truncating the input data.
158    #[doc(hidden)]
159    fn set_failed(&mut self, failed: &mut bool);
160}
161
162pub struct Position {
163    pub line: usize,
164    pub column: usize,
165}
166
167pub enum Reference<'b, 'c, T>
168where
169    T: ?Sized + 'static,
170{
171    Borrowed(&'b T),
172    Copied(&'c T),
173}
174
175impl<'b, 'c, T> Deref for Reference<'b, 'c, T>
176where
177    T: ?Sized + 'static,
178{
179    type Target = T;
180
181    fn deref(&self) -> &Self::Target {
182        match *self {
183            Reference::Borrowed(b) => b,
184            Reference::Copied(c) => c,
185        }
186    }
187}
188
189/// JSON input source that reads from a std::io input stream.
190#[cfg(feature = "std")]
191#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
192/// First byte at or after `from` that is not JSON whitespace.
193///
194/// The island when it is linked, and a plain walk when it is not, so a
195/// `--no-default-features` build still compiles and still behaves identically.
196#[cfg(feature = "std")]
197#[inline]
198fn first_non_ws_in(slice: &[u8], from: usize) -> usize {
199    #[cfg(feature = "accel")]
200    {
201        rusty_json_turbo_accel::first_non_ws(slice, from)
202    }
203    // Without the island, the crate's OWN 8-byte SWAR -- not a byte walk. The
204    // island is opt-in now, so this is the path that actually ships, and
205    // handing the reader a scalar loop here would throw away most of what
206    // brick B7 bought.
207    #[cfg(not(feature = "accel"))]
208    {
209        let mut i = from;
210        while let Some(window) = slice.get(i..i + 8) {
211            let Ok(eight) = <[u8; 8]>::try_from(window) else {
212                break;
213            };
214            let m = non_ws_bytes(u64::from_le_bytes(eight));
215            if m != 0 {
216                return i + (m.trailing_zeros() >> 3) as usize;
217            }
218            i += 8;
219        }
220        while i < slice.len() && is_ws(slice[i]) {
221            i += 1;
222        }
223        i
224    }
225}
226
227/// Bytes pulled from the reader per refill. Big enough that the per-call cost
228/// disappears, small enough to stay in L1 alongside everything else.
229#[cfg(feature = "std")]
230const IO_CHUNK: usize = 8192;
231
232/// JSON input source that reads from a std::io input stream.
233///
234/// BRICK B7. Upstream held a `LineColIterator<io::Bytes<R>>` -- one
235/// `io::Result<u8>` per byte, wrapped in a per-byte line and column
236/// counter. Measured, `from_reader` was 1.55x slower than `from_slice` on
237/// `citm_catalog` and 1.28x on `twitter`, and the gap tracked WHITESPACE
238/// exactly, because every whitespace brick this crate has landed works on
239/// a slice and this reader had none to offer.
240///
241/// Now it holds a window. The bytes are a slice, so the same scanners can
242/// run over them, and the line and column counters are computed only when
243/// an error actually needs them.
244#[cfg(feature = "std")]
245#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
246pub struct IoRead<R>
247where
248    R: io::Read,
249{
250    reader: R,
251    /// The window, allocated ONCE and never resized.
252    ///
253    /// An earlier version grew it with `resize(len + CHUNK, 0)` per refill,
254    /// which zeroed eight kilobytes every time -- as much zeroing as parsing
255    /// on a document of any size. `filled` marks how much is valid instead.
256    buf: Vec<u8>,
257    /// Bytes of `buf` holding real data; `buf[pos..filled]` is unread.
258    filled: usize,
259    pos: usize,
260    /// Absolute stream offset of `buf[0]`.
261    base: usize,
262    /// The reader has returned zero bytes.
263    eof: bool,
264    /// The peeked byte, held in a register the way upstream held it.
265    ///
266    /// This is not redundant with the buffer, and an earlier version of this
267    /// brick learned that the expensive way. Serving `peek` straight out of
268    /// the buffer means a length compare, a bounds-checked index and a store
269    /// on EVERY call -- and `canada.json` makes 2.2 million of them. That
270    /// version regressed `canada` from 1.10x to 1.32x behind `from_slice`
271    /// while improving `citm_catalog`, which is not a trade this project
272    /// accepts.
273    ///
274    /// So the hot path stays exactly upstream's shape -- one `Option` in a
275    /// register -- and the buffer sits underneath it, where the wide scanners
276    /// can reach the bytes. `pos` is already past whatever `ch` holds, which
277    /// is also what makes the position arithmetic below identical to
278    /// upstream's.
279    ch: Option<u8>,
280    /// Line number of the line containing `base`, 1-based.
281    line_at_base: usize,
282    /// Absolute offset of the first byte of the line containing `base`.
283    sol_at_base: usize,
284    /// `Some` while a `RawValue` is being captured. BRICK B13.
285    ///
286    /// It stays EMPTY: the captured bytes are taken from the window in one
287    /// copy at `end_raw_buffering`, and this is only the flag that says a
288    /// capture is running -- kept as a `Vec` so that a nested capture behaves
289    /// exactly as upstream's did.
290    #[cfg(feature = "raw_value")]
291    raw_buffer: Option<Vec<u8>>,
292    /// Index in `buf` of the first byte of the value being captured.
293    #[cfg(feature = "raw_value")]
294    raw_from: usize,
295}
296
297/// JSON input source that reads from a slice of bytes.
298//
299// This is more efficient than other iterators because peek() can be read-only
300// and we can compute line/col position only if an error happens.
301pub struct SliceRead<'a> {
302    slice: &'a [u8],
303    /// Index of the *next* byte that will be returned by next() or peek().
304    index: usize,
305    #[cfg(feature = "raw_value")]
306    raw_buffering_start_index: usize,
307}
308
309/// JSON input source that reads from a UTF-8 string.
310//
311// Able to elide UTF-8 checks by assuming that the input is valid UTF-8.
312pub struct StrRead<'a> {
313    delegate: SliceRead<'a>,
314    #[cfg(feature = "raw_value")]
315    data: &'a str,
316}
317
318// Prevent users from implementing the Read trait.
319mod private {
320    pub trait Sealed {}
321}
322
323//////////////////////////////////////////////////////////////////////////////
324
325#[cfg(feature = "std")]
326impl<R> IoRead<R>
327where
328    R: io::Read,
329{
330    /// Create a JSON input source to read from a std::io input stream.
331    ///
332    /// **This buffers internally, so you do not need a
333    /// [`std::io::BufReader`].** Upstream did not, and told you to add one;
334    /// this reads in 8 KiB blocks itself, so wrapping the input in a second
335    /// buffer only adds a layer. It is harmless on a [`File`] and a measurable
336    /// loss on data already in memory.
337    ///
338    /// For bytes you already hold, [`from_slice`](crate::from_slice) is still
339    /// the faster route -- it can borrow strings straight out of the input,
340    /// which no reader can do.
341    ///
342    /// [`File`]: std::fs::File
343    pub fn new(reader: R) -> Self {
344        IoRead {
345            reader,
346            buf: ::alloc::vec::from_elem(0, IO_CHUNK * 2)alloc::vec![0; IO_CHUNK * 2],
347            filled: 0,
348            pos: 0,
349            base: 0,
350            eof: false,
351            ch: None,
352            line_at_base: 1,
353            sol_at_base: 0,
354            #[cfg(feature = "raw_value")]
355            raw_buffer: None,
356            #[cfg(feature = "raw_value")]
357            raw_from: 0,
358        }
359    }
360
361    /// Drop what has been consumed and pull more bytes.
362    ///
363    /// Only called when the window is exhausted, so nothing peeked is ever
364    /// dropped: a peeked byte is by definition still in the window.
365    // COLD, and marked so deliberately: a refill happens once per 8 KiB, so
366    // inlining it into `pull` would put its `memchr`, its `copy_within` and its
367    // `read` call in the middle of a path taken once per BYTE. `canada.json`
368    // makes 2.2 million of those.
369    #[cold]
370    #[inline(never)]
371    fn fill(&mut self) -> Result<()> {
372        // BRICK B13. While a `RawValue` is being captured its bytes have to
373        // stay addressable, so the window is drained only as far as where the
374        // capture STARTS rather than as far as `pos`. Everything before
375        // `raw_from` is consumed and unreachable, so dropping that much is
376        // free, and what is held is bounded by the size of the value -- which
377        // is exactly what upstream's raw buffer held anyway.
378        #[cfg(feature = "raw_value")]
379        let drop_to = if self.raw_buffer.is_some() {
380            // The whole prefix before the capture goes, so `raw_from` lands
381            // on 0 -- which `take` does in the same move as reading it.
382            core::mem::take(&mut self.raw_from)
383        } else {
384            self.pos
385        };
386        #[cfg(not(feature = "raw_value"))]
387        let drop_to = self.pos;
388
389        if drop_to > 0 {
390            // The bytes about to be dropped are the only place the line
391            // counters can be advanced from, so roll them forward first.
392            self.roll_lines(drop_to);
393            self.buf.copy_within(drop_to..self.filled, 0);
394            self.filled -= drop_to;
395            self.base += drop_to;
396            self.pos -= drop_to;
397        }
398        if self.eof {
399            return Ok(());
400        }
401        if self.filled == self.buf.len() {
402            // A full window with nothing droppable is only reachable while
403            // capturing a `RawValue` whose text is longer than the buffer:
404            // otherwise the drain above frees the whole consumed prefix, and
405            // `fill` is only ever called with the window spent. Grow instead
406            // of sliding, so the captured bytes stay put; doubling keeps the
407            // number of growths logarithmic in the size of the value.
408            #[cfg(feature = "raw_value")]
409            {
410                if true {
    if !self.raw_buffer.is_some() {
        {
            ::core::panicking::panic_fmt(format_args!("the window can only be full with nothing to drop while capturing"));
        }
    };
};debug_assert!(
411                    self.raw_buffer.is_some(),
412                    "the window can only be full with nothing to drop while capturing"
413                );
414                crate::counters::add(&crate::counters::RAW_GROWS, 1);
415                self.buf.resize(self.buf.len() * 2, 0);
416            }
417            #[cfg(not(feature = "raw_value"))]
418            return Ok(());
419        }
420        // A short read is fine and expected from a pipe or socket: the caller
421        // asks again, and `eof` is set only by a read of length zero.
422        match self.reader.read(&mut self.buf[self.filled..]) {
423            Ok(0) => {
424                self.eof = true;
425                Ok(())
426            }
427            Ok(n) => {
428                self.filled += n;
429                Ok(())
430            }
431            Err(e) => Err(Error::io(e)),
432        }
433    }
434
435    /// Advance `line_at_base` / `sol_at_base` over `buf[..n]`.
436    ///
437    /// `memchr` rather than a per-byte branch, which is the other half of what
438    /// B7 removes: upstream counted lines on every byte whether or not any
439    /// error was ever reported.
440    fn roll_lines(&mut self, n: usize) {
441        let seg = &self.buf[..n];
442        let mut count = 0;
443        let mut last = None;
444        for i in memchr::memchr_iter(b'\n', seg) {
445            count += 1;
446            last = Some(i);
447        }
448        if let Some(i) = last {
449            self.sol_at_base = self.base + i + 1;
450        }
451        self.line_at_base += count;
452    }
453
454    /// Line and column at absolute offset `k`, matching `LineColIterator`
455    /// exactly: line is 1-based and counts newlines PULLED, column is the
456    /// number of bytes since the start of the current line, so it is 0
457    /// immediately after a newline.
458    ///
459    /// Only reached from an error path, so scanning the window here is free in
460    /// the sense that matters.
461    fn position_at(&self, k: usize) -> Position {
462        let rel = (k - self.base).min(self.filled);
463        let mut line = self.line_at_base;
464        let mut sol = self.sol_at_base;
465        for i in memchr::memchr_iter(b'\n', &self.buf[..rel]) {
466            line += 1;
467            sol = self.base + i + 1;
468        }
469        Position {
470            line,
471            column: k - sol,
472        }
473    }
474
475    /// Bytes PULLED from the stream. `pos` is already past a byte held in
476    /// `ch`, so this is exactly what `LineColIterator::byte_offset` reported.
477    fn pulled(&self) -> usize {
478        self.base + self.pos
479    }
480
481    /// Pull one byte into `ch`, refilling if the window is spent.
482    #[inline(always)]
483    fn pull(&mut self) -> Result<Option<u8>> {
484        if self.pos == self.filled {
485            match self.fill() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(self.fill());
486            if self.pos == self.filled {
487                return Ok(None);
488            }
489        }
490        let ch = self.buf[self.pos];
491        self.pos += 1;
492        Ok(Some(ch))
493    }
494}
495
496#[cfg(feature = "std")]
497impl<R> private::Sealed for IoRead<R> where R: io::Read {}
498
499#[cfg(feature = "std")]
500impl<R> IoRead<R>
501where
502    R: io::Read,
503{
504    fn parse_str_bytes<'s, T, F>(
505        &'s mut self,
506        scratch: &'s mut Vec<u8>,
507        validate: bool,
508        result: F,
509    ) -> Result<T>
510    where
511        T: 's,
512        F: FnOnce(&'s Self, &'s [u8]) -> Result<T>,
513    {
514        loop {
515            let ch = match next_or_eof(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
516            if !is_escape(ch, true) {
517                scratch.push(ch);
518                continue;
519            }
520            match ch {
521                b'"' => {
522                    return result(self, scratch);
523                }
524                b'\\' => {
525                    match parse_escape(self, validate, scratch) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(parse_escape(self, validate, scratch));
526                }
527                _ => {
528                    if validate {
529                        return error(self, ErrorCode::ControlCharacterWhileParsingString);
530                    }
531                    scratch.push(ch);
532                }
533            }
534        }
535    }
536}
537
538#[cfg(feature = "std")]
539impl<'de, R> Read<'de> for IoRead<R>
540where
541    R: io::Read,
542{
543    #[inline]
544    fn next(&mut self) -> Result<Option<u8>> {
545        // No `RawValue` push here: brick B13 takes the captured text as one
546        // range from the window, so this is the same two instructions whether
547        // or not the `raw_value` feature is on.
548        match self.ch.take() {
549            Some(ch) => Ok(Some(ch)),
550            None => self.pull(),
551        }
552    }
553
554    #[inline]
555    fn peek(&mut self) -> Result<Option<u8>> {
556        match self.ch {
557            Some(ch) => Ok(Some(ch)),
558            None => {
559                self.ch = match self.pull() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(self.pull());
560                Ok(self.ch)
561            }
562        }
563    }
564
565    #[inline]
566    fn discard(&mut self) {
567        self.ch = None;
568    }
569
570    /// THE POINT OF BRICK B7: run the wide whitespace scanner over the window.
571    ///
572    /// This is the half of the ceiling the buffer exists to unlock. Measured,
573    /// the 1.55x gap between `from_reader` and `from_slice` on `citm_catalog`
574    /// decomposed as roughly 1.41x from the missing slice scanners and only
575    /// 1.10x from the per-byte iterator -- so a buffer WITHOUT this override is
576    /// all cost and no benefit, which an earlier version of this brick duly
577    /// measured as a regression on every file.
578    ///
579    /// A whitespace run may cross a refill, so the scan repeats until it finds
580    /// a non-whitespace byte or the reader is spent. The byte is left in `ch`
581    /// rather than consumed, exactly as the trait's `peek`-based default leaves
582    /// it, so error positions and `RawValue` capture are unchanged.
583    #[inline]
584    fn skip_whitespace(&mut self) -> Result<Option<u8>> {
585        // Whatever was already peeked comes first, or the scan would step over
586        // it and lose a byte.
587        if let Some(c) = self.ch {
588            if !is_ws(c) {
589                return Ok(Some(c));
590            }
591            self.ch = None;
592        }
593        // THE TINY ENTRY, and this is the third time this project has needed
594        // one. On a minified document there is no whitespace at all, so every
595        // call would otherwise enter the scanner, dispatch on the ISA, and
596        // find its answer at offset zero. `canada.json` and
597        // `s4-frame-telemetry.json` regressed 1.10x -> 1.16x and 1.02x ->
598        // 1.10x on exactly that before this existed.
599        //
600        // Brick B1s needed a 4-byte scalar peel for the same reason, and M3's
601        // escape scanner needed a length guard. The law is the same each time:
602        // a wide scanner must not be reached until something cheap has ruled
603        // out the common case.
604        if self.pos < self.filled {
605            let c = self.buf[self.pos];
606            if !is_ws(c) {
607                self.pos += 1;
608                self.ch = Some(c);
609                return Ok(Some(c));
610            }
611        }
612        loop {
613            let at = first_non_ws_in(&self.buf[..self.filled], self.pos);
614            // A `RawValue` captures the raw TEXT of a value, whitespace and
615            // all -- `{"foo": 2}` must not come back as `{"foo":2}`, which is
616            // what upstream's `test_boxed_raw_value` caught when an earlier
617            // version of this override skipped a run without recording it.
618            // Nothing is recorded here any more: brick B13 takes the whole
619            // range at the end, whitespace included.
620            if at < self.filled {
621                // Pull it: `pos` moves past the byte and `ch` holds it, which
622                // is the state a `peek` would have left.
623                self.pos = at + 1;
624                let c = self.buf[at];
625                self.ch = Some(c);
626                return Ok(Some(c));
627            }
628            self.pos = self.filled;
629            match self.fill() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(self.fill());
630            if self.pos == self.filled {
631                return Ok(None);
632            }
633        }
634    }
635
636    fn position(&self) -> Position {
637        self.position_at(self.pulled())
638    }
639
640    fn peek_position(&self) -> Position {
641        // Upstream's comment: "The LineColIterator updates its position during
642        // peek() so it has the right one here." A peek here pulls too, so the
643        // same is true and these stay equal.
644        self.position()
645    }
646
647    fn byte_offset(&self) -> usize {
648        // Upstream computed `iter.byte_offset() - 1` when a byte was peeked.
649        // Same arithmetic, same meaning: bytes CONSUMED.
650        match self.ch {
651            Some(_) => self.pulled() - 1,
652            None => self.pulled(),
653        }
654    }
655
656    fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'de, 's, str>> {
657        self.parse_str_bytes(scratch, true, as_str)
658            .map(Reference::Copied)
659    }
660
661    fn parse_str_raw<'s>(
662        &'s mut self,
663        scratch: &'s mut Vec<u8>,
664    ) -> Result<Reference<'de, 's, [u8]>> {
665        self.parse_str_bytes(scratch, false, |_, bytes| Ok(bytes))
666            .map(Reference::Copied)
667    }
668
669    fn ignore_str(&mut self) -> Result<()> {
670        loop {
671            let ch = match next_or_eof(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
672            if !is_escape(ch, true) {
673                continue;
674            }
675            match ch {
676                b'"' => {
677                    return Ok(());
678                }
679                b'\\' => {
680                    match ignore_escape(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(ignore_escape(self));
681                }
682                _ => {
683                    return error(self, ErrorCode::ControlCharacterWhileParsingString);
684                }
685            }
686        }
687    }
688
689    fn decode_hex_escape(&mut self) -> Result<u16> {
690        let a = match next_or_eof(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
691        let b = match next_or_eof(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
692        let c = match next_or_eof(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
693        let d = match next_or_eof(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(self));
694        match decode_four_hex_digits(a, b, c, d) {
695            Some(val) => Ok(val),
696            None => error(self, ErrorCode::InvalidEscape),
697        }
698    }
699
700    /// BRICK B13: a `RawValue` over a reader as a RANGE, not a byte stream.
701    ///
702    /// Upstream captured a `RawValue` by pushing every byte into a `Vec` as it
703    /// was consumed, through `next` and `discard`. Priced on a 10,000-document
704    /// log stream, `RawValue` over a reader ran at 257 MB/s against **1,074
705    /// MB/s for the same values over a slice** -- 4.18x -- and sat at parity
706    /// with upstream (1.009x), because the per-byte push, not the parse, was
707    /// the cost.
708    ///
709    /// The bytes are already contiguous in the window, so all this has to
710    /// record is where the value starts; `fill` then holds the window from
711    /// there and `end_raw_buffering` takes it in one copy.
712    ///
713    /// A byte sitting in `ch` has been PULLED but not CONSUMED, and upstream
714    /// pushed it only if it was later consumed -- so the start is `pos` less
715    /// that byte, which is the same arithmetic as `byte_offset`.
716    #[cfg(feature = "raw_value")]
717    fn begin_raw_buffering(&mut self) {
718        self.raw_from = self.pos - usize::from(self.ch.is_some());
719        self.raw_buffer = Some(Vec::new());
720    }
721
722    #[cfg(feature = "raw_value")]
723    fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
724    where
725        V: Visitor<'de>,
726    {
727        // Whatever is in `ch` was peeked to END the value and is not part of
728        // it -- a number stops on the byte after its last digit.
729        let end = self.pos - usize::from(self.ch.is_some());
730        let mut raw = self.raw_buffer.take().unwrap();
731        // ONE copy, of exactly the right size: `extend_from_slice` on an empty
732        // `Vec` reserves the slice's length, and `String::from_utf8` then
733        // takes the allocation as it stands.
734        crate::counters::add(&crate::counters::RAW_CAPTURES, 1);
735        crate::counters::add(&crate::counters::RAW_BYTES, (end - self.raw_from) as u64);
736        raw.extend_from_slice(&self.buf[self.raw_from..end]);
737        // Give any growth back. A value longer than the window grew `buf`,
738        // and a stream carrying one enormous value must not go on paying for
739        // it on every document after -- upstream allocated a fresh `Vec` per
740        // value and freed it each time, and this has to be no worse. In the
741        // common case the window never grew and this is one length compare.
742        if self.buf.len() > IO_CHUNK * 2 && self.filled - self.pos <= IO_CHUNK * 2 {
743            self.roll_lines(self.pos);
744            self.buf.copy_within(self.pos..self.filled, 0);
745            self.filled -= self.pos;
746            self.base += self.pos;
747            self.pos = 0;
748            self.buf.truncate(IO_CHUNK * 2);
749            self.buf.shrink_to_fit();
750            crate::counters::add(&crate::counters::RAW_SHRINKS, 1);
751        }
752        let raw = match String::from_utf8(raw) {
753            Ok(raw) => raw,
754            Err(_) => return error(self, ErrorCode::InvalidUnicodeCodePoint),
755        };
756        visitor.visit_map(OwnedRawDeserializer {
757            raw_value: Some(raw),
758        })
759    }
760
761    const should_early_return_if_failed: bool = true;
762
763    #[inline]
764    #[cold]
765    fn set_failed(&mut self, failed: &mut bool) {
766        *failed = true;
767    }
768}
769
770//////////////////////////////////////////////////////////////////////////////
771
772impl<'a> SliceRead<'a> {
773    /// Create a JSON input source to read from a slice of bytes.
774    pub fn new(slice: &'a [u8]) -> Self {
775        SliceRead {
776            slice,
777            index: 0,
778            #[cfg(feature = "raw_value")]
779            raw_buffering_start_index: 0,
780        }
781    }
782
783    fn position_of_index(&self, i: usize) -> Position {
784        let start_of_line = match memchr::memrchr(b'\n', &self.slice[..i]) {
785            Some(position) => position + 1,
786            None => 0,
787        };
788        Position {
789            line: 1 + memchr::memchr_iter(b'\n', &self.slice[..start_of_line]).count(),
790            column: i - start_of_line,
791        }
792    }
793
794    /// PROBE WRAPPER (brick B2). Counts one run and the bytes it advanced
795    /// over, for every caller, without touching the scanner itself.
796    /// `inline(always)` on a body that compiles to nothing without `profile`,
797    /// so a shipping build is byte-for-byte what it was.
798    #[inline(always)]
799    fn skip_to_escape(&mut self, forbid_control_characters: bool) {
800        let from = self.index;
801        self.skip_to_escape_impl(forbid_control_characters);
802        crate::counters::add(&crate::counters::STR_SCANS, 1);
803        crate::counters::add(&crate::counters::STR_BYTES, (self.index - from) as u64);
804    }
805
806    fn skip_to_escape_impl(&mut self, forbid_control_characters: bool) {
807        // Immediately bail-out on empty strings and consecutive escapes (e.g. \u041b\u0435)
808        if self.index == self.slice.len()
809            || is_escape(self.slice[self.index], forbid_control_characters)
810        {
811            return;
812        }
813        self.index += 1;
814
815        let rest = &self.slice[self.index..];
816
817        if !forbid_control_characters {
818            self.index += memchr::memchr2(b'"', b'\\', rest).unwrap_or(rest.len());
819            return;
820        }
821
822        // We wish to find the first byte in range 0x00..=0x1F or " or \. Ideally, we'd use
823        // something akin to memchr3, but the memchr crate does not support this at the moment.
824        // Therefore, we use a variation on Mycroft's algorithm [1] to provide performance better
825        // than a naive loop. It runs faster than equivalent two-pass memchr2+SWAR code on
826        // benchmarks and it's cross-platform, so probably the right fit.
827        // [1]: https://groups.google.com/forum/#!original/comp.lang.c/2HtQXvg7iKc/xOJeipH6KLMJ
828
829        #[cfg(fast_arithmetic = "64")]
830        type Chunk = u64;
831        #[cfg(fast_arithmetic = "32")]
832        type Chunk = u32;
833
834        const STEP: usize = mem::size_of::<Chunk>();
835        const ONE_BYTES: Chunk = Chunk::MAX / 255; // 0x0101...01
836
837        for chunk in rest.chunks_exact(STEP) {
838            let chars = Chunk::from_le_bytes(chunk.try_into().unwrap());
839            let contains_ctrl = chars.wrapping_sub(ONE_BYTES * 0x20) & !chars;
840            let chars_quote = chars ^ (ONE_BYTES * Chunk::from(b'"'));
841            let contains_quote = chars_quote.wrapping_sub(ONE_BYTES) & !chars_quote;
842            let chars_backslash = chars ^ (ONE_BYTES * Chunk::from(b'\\'));
843            let contains_backslash = chars_backslash.wrapping_sub(ONE_BYTES) & !chars_backslash;
844            let masked = (contains_ctrl | contains_quote | contains_backslash) & (ONE_BYTES << 7);
845            if masked != 0 {
846                // SAFETY: chunk is in-bounds for slice
847                self.index = unsafe { chunk.as_ptr().offset_from(self.slice.as_ptr()) } as usize
848                    + masked.trailing_zeros() as usize / 8;
849                return;
850            }
851        }
852
853        self.index += rest.len() / STEP * STEP;
854        self.skip_to_escape_slow();
855    }
856
857    #[cold]
858    #[inline(never)]
859    fn skip_to_escape_slow(&mut self) {
860        while self.index < self.slice.len() && !is_escape(self.slice[self.index], true) {
861            self.index += 1;
862        }
863    }
864
865    /// The big optimization here over IoRead is that if the string contains no
866    /// backslash escape sequences, the returned &str is a slice of the raw JSON
867    /// data so we avoid copying into the scratch space.
868    fn parse_str_bytes<'s, T, F>(
869        &'s mut self,
870        scratch: &'s mut Vec<u8>,
871        validate: bool,
872        result: F,
873    ) -> Result<Reference<'a, 's, T>>
874    where
875        T: ?Sized + 's,
876        F: for<'f> FnOnce(&'s Self, &'f [u8]) -> Result<&'f T>,
877    {
878        // Index of the first byte not yet copied into the scratch space.
879        let mut start = self.index;
880
881        loop {
882            self.skip_to_escape(validate);
883            if self.index == self.slice.len() {
884                return error(self, ErrorCode::EofWhileParsingString);
885            }
886            match self.slice[self.index] {
887                b'"' => {
888                    if scratch.is_empty() {
889                        // Fast path: return a slice of the raw JSON without any
890                        // copying.
891                        crate::counters::add(&crate::counters::STR_BORROWS, 1);
892                        let borrowed = &self.slice[start..self.index];
893                        self.index += 1;
894                        return result(self, borrowed).map(Reference::Borrowed);
895                    } else {
896                        crate::counters::add(&crate::counters::STR_COPIES, 1);
897                        scratch.extend_from_slice(&self.slice[start..self.index]);
898                        self.index += 1;
899                        return result(self, scratch).map(Reference::Copied);
900                    }
901                }
902                b'\\' => {
903                    scratch.extend_from_slice(&self.slice[start..self.index]);
904                    self.index += 1;
905                    match parse_escape(self, validate, scratch) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(parse_escape(self, validate, scratch));
906                    start = self.index;
907                }
908                _ => {
909                    self.index += 1;
910                    return error(self, ErrorCode::ControlCharacterWhileParsingString);
911                }
912            }
913        }
914    }
915}
916
917// ---------------------------------------------------------------------------
918// Whitespace scanning.
919//
920// Insignificant JSON whitespace is exactly ` `, `\n`, `\t`, `\r`. Every other
921// byte <= 0x20 is invalid where whitespace is allowed and must be RETURNED, not
922// skipped, so the parser reports it at the right line and column. That rules
923// out the tempting `b <= 0x20` test and any range test over 0x09..=0x0D, which
924// would silently accept the vertical tab and form feed.
925//
926// Why a wide scan is worth it here, measured rather than assumed: on
927// `citm_catalog.json` **98.0% of whitespace bytes live in runs of 8 or more**,
928// and 92.3% in runs of 17 to 32 -- deep indentation, one run per line. On
929// `twitter.json` it is 92.6% in runs of 8 or more. A per-byte loop walks all of
930// it one compare at a time. (The run *mean* alone would have said the opposite;
931// the distribution is what decides, and the two differ because most calls to
932// `skip_whitespace` find no whitespace at all.)
933// ---------------------------------------------------------------------------
934
935// The SWAR primitives below serve the non-`accel` build only; with the island
936// on, the same exact predicates live in `rusty_json_turbo-accel`, tested there
937// against the same oracle.
938#[cfg(not(feature = "accel"))]
939const WS_ONES: u64 = 0x0101_0101_0101_0101;
940#[cfg(not(feature = "accel"))]
941const WS_HIGHS: u64 = 0x8080_8080_8080_8080;
942
943#[inline(always)]
944fn is_ws(b: u8) -> bool {
945    #[allow(non_exhaustive_omitted_patterns)] match b {
    b' ' | b'\n' | b'\t' | b'\r' => true,
    _ => false,
}matches!(b, b' ' | b'\n' | b'\t' | b'\r')
946}
947
948/// High bit set in each byte of `x` that is zero, and clear in every other.
949///
950/// Exact per byte, unlike the classic `(x - ONES) & !x & HIGHS`, whose
951/// subtraction borrows across byte boundaries and reports a byte as zero
952/// because its neighbour was. Here the addition is `low7 + 0x7F <= 0xFE`, which
953/// cannot carry out of its own byte, so no byte can be contaminated.
954#[cfg(not(feature = "accel"))]
955#[inline(always)]
956fn zero_bytes(x: u64) -> u64 {
957    // `low7 + 0x7F` sets bit 7 iff low7 != 0; OR the original bit 7 back in, so
958    // the high bit marks a NON-zero byte. Invert for the zero bytes.
959    !(((x & !WS_HIGHS).wrapping_add(!WS_HIGHS)) | x) & WS_HIGHS
960}
961
962/// High bit set in each byte of `x` equal to `c`.
963#[cfg(not(feature = "accel"))]
964#[inline(always)]
965fn eq_bytes(x: u64, c: u8) -> u64 {
966    zero_bytes(x ^ (c as u64).wrapping_mul(WS_ONES))
967}
968
969/// High bit set in each byte of `x` that is NOT insignificant whitespace.
970#[cfg(not(feature = "accel"))]
971#[inline(always)]
972fn non_ws_bytes(x: u64) -> u64 {
973    let ws = eq_bytes(x, b' ') | eq_bytes(x, b'\n') | eq_bytes(x, b'\t') | eq_bytes(x, b'\r');
974    !ws & WS_HIGHS
975}
976
977// Both scanners return the byte they stopped on as well as its index.
978//
979// Returning only the index looked tidier and cost a measured 4% on
980// `canada.json`: the caller had to load that byte a second time, on every one
981// of the 557,593 calls a whitespace-free document makes. The file that CANNOT
982// benefit from a whitespace change is the one that exposed it -- which is the
983// whole reason to keep such a file in the corpus and to require that it reads
984// exactly the old number.
985
986/// The oracle: the first byte at or after `from` that is not insignificant
987/// whitespace, with its index. One byte at a time.
988///
989/// Stays in the tree forever. It is what the wide scan is gated against, and
990/// what runs when the knob turns the wide path off.
991#[inline]
992// THE ORACLE STAYS, even when nothing calls it in a default build. With the
993// island linked, the wide path lives in `rusty_json_turbo-accel` and reaches
994// its OWN scalar twin through `RJT_ISA=scalar`, which leaves this one reachable
995// only from the `knobs` A/B and the tests below. It is kept regardless: a fast
996// path whose scalar twin has been deleted cannot be checked and cannot be
997// debugged, and this crate's tests compare against exactly this function.
998#[allow(dead_code)]
999fn scan_ws_scalar(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1000    let mut i = from;
1001    while i < slice.len() {
1002        let b = slice[i];
1003        if !is_ws(b) {
1004            return (i, Some(b));
1005        }
1006        i += 1;
1007    }
1008    (i, None)
1009}
1010
1011/// How many bytes to walk one at a time before reaching for a wide step.
1012///
1013/// Not a guess. The run-length census says **46% of `twitter.json`'s whitespace
1014/// runs are a single byte** (13,345 of 28,826) and 34% of `citm_catalog`'s are.
1015/// Paying an eight-byte load and a SWAR test to discover that costs far more
1016/// than the two compares it replaces, and measured **0.903x on twitter scan,
1017/// 20/21** -- a real regression on a file with plenty of whitespace. Peeling a
1018/// few bytes first means a short run never reaches the wide path, while a long
1019/// one pays four extra compares out of seventeen or more.
1020const WS_PEEL: usize = 4;
1021
1022/// Skip a whitespace run that is already known to have started, and return the
1023/// byte that ended it.
1024///
1025/// Deliberately NOT `inline(always)`: it runs only when there is whitespace to
1026/// skip, so inlining it into every call site would bloat the hot path that
1027/// mostly finds no whitespace at all.
1028#[inline]
1029fn scan_ws_run(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1030    #[cfg(feature = "knobs")]
1031    if !ws_wide() {
1032        return scan_ws_scalar(slice, from);
1033    }
1034
1035    let mut i = from;
1036    // Peel a short run without touching the wide path (see `WS_PEEL`).
1037    //
1038    // This still matters, and matters MORE the wider the step gets:
1039    // `twitter`'s whitespace run is a single byte 13,346 times, and an 8-byte
1040    // step over those cost 10% before this peel existed.
1041    let peel = (from + WS_PEEL).min(slice.len());
1042    while i < peel {
1043        let b = slice[i];
1044        if !is_ws(b) {
1045            return (i, Some(b));
1046        }
1047        i += 1;
1048    }
1049    scan_ws_wide(slice, i)
1050}
1051
1052/// The wide scan, handed to the SIMD island (M3).
1053///
1054/// `rusty_json_turbo-accel` runs the same predicate 16 or 32 bytes at a time
1055/// on x86-64, with the scalar walk as its permanent oracle and `RJT_ISA` to
1056/// choose a rung at runtime. It is a separate crate so that no `unsafe` for a
1057/// vector load is ever written in the parser.
1058#[cfg(feature = "accel")]
1059#[inline]
1060fn scan_ws_wide(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1061    let at = rusty_json_turbo_accel::first_non_ws(slice, from);
1062    // Return the byte with the index. Returning only the index cost a measured
1063    // 4% on `canada.json`, because the caller loaded it again.
1064    (at, slice.get(at).copied())
1065}
1066
1067/// The wide scan without the island: 8 bytes per step, SWAR.
1068///
1069/// Written so there is no panicking branch at all: no indexing, no unwrap, no
1070/// unreachable. Both fallible steps are provably infallible here and fold
1071/// away, and if either ever were not, the scalar tail finishes the job
1072/// correctly rather than aborting.
1073#[cfg(not(feature = "accel"))]
1074fn scan_ws_wide(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1075    let mut i = from;
1076    while let Some(window) = slice.get(i..i + 8) {
1077        let Ok(bytes) = <[u8; 8]>::try_from(window) else {
1078            break;
1079        };
1080        let chunk = u64::from_le_bytes(bytes);
1081        let m = non_ws_bytes(chunk);
1082        if m != 0 {
1083            // Little-endian: byte 0 is the lowest address, so the first set
1084            // high bit marks the first non-whitespace byte. Take the byte out
1085            // of the word already in hand rather than loading it again.
1086            let k = (m.trailing_zeros() >> 3) as usize;
1087            return (i + k, Some((chunk >> (k * 8)) as u8));
1088        }
1089        i += 8;
1090    }
1091    scan_ws_scalar(slice, i)
1092}
1093
1094// ---------------------------------------------------------------------------
1095// Digit runs.
1096// ---------------------------------------------------------------------------
1097
1098/// Are all eight bytes ASCII digits?
1099///
1100/// For a digit `c` in `0x30..=0x39`: `c & 0xF0 == 0x30`, and `c + 6` stays
1101/// inside `0x36..=0x3F` so `(c + 6) & 0xF0 == 0x30` too, giving `0x33` once the
1102/// second nibble is folded down. Any other byte fails its OWN test -- a
1103/// non-digit can carry into its neighbour, but it can never rescue itself, so
1104/// the chunk is always rejected and a false accept is impossible.
1105#[inline(always)]
1106fn all_8_digits(chunk: u64) -> bool {
1107    const LO: u64 = 0x0606_0606_0606_0606;
1108    const HI: u64 = 0xF0F0_F0F0_F0F0_F0F0;
1109    const WANT: u64 = 0x3333_3333_3333_3333;
1110    ((chunk & HI) | (((chunk.wrapping_add(LO)) & HI) >> 4)) == WANT
1111}
1112
1113/// The value of eight ASCII digits, first character most significant.
1114///
1115/// The standard three-multiply fold (as in fast_float and simdjson). `chunk`
1116/// must have come from a little-endian load, so the first character is the
1117/// lowest byte, and every byte must be a digit -- check [`all_8_digits`] first.
1118/// Every step is deliberately WRAPPING. The intermediate products overflow
1119/// `u64` by design and the answer is read out of the high half, so plain `*`
1120/// would be correct in a release build and **panic in a debug build** -- a
1121/// consumer-visible bug that only shows up off the happy path. The unit test
1122/// below runs in debug, which is how this was caught.
1123#[inline(always)]
1124fn eight_digits_value(chunk: u64) -> u32 {
1125    const MASK: u64 = 0x0000_00FF_0000_00FF;
1126    const MUL1: u64 = 0x000F_4240_0000_0064; // 100 + (1_000_000 << 32)
1127    const MUL2: u64 = 0x0000_2710_0000_0001; // 1 + (10_000 << 32)
1128    let v = chunk.wrapping_sub(0x3030_3030_3030_3030);
1129    let v = v.wrapping_mul(10).wrapping_add(v >> 8);
1130    let lo = (v & MASK).wrapping_mul(MUL1);
1131    let hi = ((v >> 16) & MASK).wrapping_mul(MUL2);
1132    (lo.wrapping_add(hi) >> 32) as u32
1133}
1134
1135// A FOUR-digit step was built on top of this one, twice, and refuted twice.
1136// It is not here, and the reason is a law rather than an accident -- see
1137// `parse_decimal` in `de.rs` and brick B4 in `corpus/LEDGER.md`.
1138
1139/// Is the wide digit scan on? `knobs` builds only.
1140#[cfg(feature = "knobs")]
1141fn num_wide() -> bool {
1142    static V: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1143    *V.get_or_init(|| std::env::var("RJT_NUM_WIDE").map_or(true, |v| v != "0"))
1144}
1145
1146/// Is the wide whitespace scan on? `knobs` builds only.
1147#[cfg(feature = "knobs")]
1148fn ws_wide() -> bool {
1149    static V: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1150    *V.get_or_init(|| std::env::var("RJT_WS_WIDE").map_or(true, |v| v != "0"))
1151}
1152
1153/// The first non-whitespace byte at or after `from`, with its index.
1154///
1155/// `inline(always)`, and tiny on purpose. The overwhelmingly common case is
1156/// that there is no whitespace to skip at all -- `canada.json` takes it on
1157/// essentially all of its 557,593 calls, `citm_catalog` on 92,950 of 169,287 --
1158/// and that case must cost exactly what the original `peek()` cost: one load,
1159/// one test. Everything else is behind a call.
1160#[inline(always)]
1161fn scan_ws(slice: &[u8], from: usize) -> (usize, Option<u8>) {
1162    match slice.get(from) {
1163        Some(&b) if !is_ws(b) => (from, Some(b)),
1164        None => (from, None),
1165        _ => scan_ws_run(slice, from),
1166    }
1167}
1168
1169/// Is the whitespace fast path on? `knobs` builds only; cached, so the
1170/// environment is read once per process rather than once per whitespace run.
1171///
1172/// This is the A/B instrument for the whitespace brick: one binary, one code
1173/// layout, one env var between the arms. Comparing two *builds* instead would
1174/// confound the change with its layout, and layout alone is worth +-13% here.
1175#[cfg(feature = "knobs")]
1176fn ws_fastpath() -> bool {
1177    static V: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
1178    *V.get_or_init(|| std::env::var("RJT_WS_FASTPATH").map_or(true, |v| v != "0"))
1179}
1180
1181impl<'a> private::Sealed for SliceRead<'a> {}
1182
1183impl<'a> Read<'a> for SliceRead<'a> {
1184    #[inline]
1185    fn next(&mut self) -> Result<Option<u8>> {
1186        // `Ok(self.slice.get(self.index).map(|ch| { self.index += 1; *ch }))`
1187        // is about 10% slower.
1188        crate::counters::add(&crate::counters::NEXT, 1);
1189        Ok(if self.index < self.slice.len() {
1190            let ch = self.slice[self.index];
1191            self.index += 1;
1192            Some(ch)
1193        } else {
1194            None
1195        })
1196    }
1197
1198    #[inline]
1199    fn peek(&mut self) -> Result<Option<u8>> {
1200        // `Ok(self.slice.get(self.index).map(|ch| *ch))` is about 10% slower
1201        // for some reason.
1202        crate::counters::add(&crate::counters::PEEK, 1);
1203        Ok(if self.index < self.slice.len() {
1204            Some(self.slice[self.index])
1205        } else {
1206            None
1207        })
1208    }
1209
1210    #[inline]
1211    fn discard(&mut self) {
1212        crate::counters::add(&crate::counters::DISCARD, 1);
1213        self.index += 1;
1214    }
1215
1216    #[inline]
1217    fn take_8_digits(&mut self) -> Option<u32> {
1218        #[cfg(feature = "knobs")]
1219        if !num_wide() {
1220            return None;
1221        }
1222        // No `?`: the crate denies `clippy::question_mark_used`.
1223        let Some(window) = self.slice.get(self.index..self.index + 8) else {
1224            return None;
1225        };
1226        let Ok(bytes) = <[u8; 8]>::try_from(window) else {
1227            return None;
1228        };
1229        let chunk = u64::from_le_bytes(bytes);
1230        crate::counters::add(&crate::counters::D8_CALLS, 1);
1231        if all_8_digits(chunk) {
1232            crate::counters::add(&crate::counters::D8_HITS, 1);
1233            self.index += 8;
1234            Some(eight_digits_value(chunk))
1235        } else {
1236            None
1237        }
1238    }
1239
1240    #[inline]
1241    fn skip_whitespace(&mut self) -> Result<Option<u8>> {
1242        // MEASUREMENT ARM, `profile` builds only: `RJT_WS_FASTPATH=0` forces the
1243        // old byte-at-a-time route so both arms can be COUNTED in one binary.
1244        // The knob is resolved once and cached, and it is read once per
1245        // whitespace run rather than per byte, so it does not itself become the
1246        // thing being measured.
1247        #[cfg(feature = "knobs")]
1248        if !ws_fastpath() {
1249            loop {
1250                match tri!(self.peek()) {
1251                    Some(b' ' | b'\n' | b'\t' | b'\r') => self.discard(),
1252                    other => return Ok(other),
1253                }
1254            }
1255        }
1256
1257        // One walk of the slice, eight bytes at a time where the run is long
1258        // enough, instead of a `Result<Option<u8>>` round trip per byte through
1259        // peek()/discard(). Byte-for-byte the same decision as the default
1260        // loop: exactly the four insignificant whitespace bytes are skipped,
1261        // and the first byte that is not one of them is returned WITHOUT being
1262        // consumed.
1263        let slice = self.slice;
1264        let start = self.index;
1265        let (i, next) = scan_ws(slice, start);
1266        self.index = i;
1267        crate::counters::add(&crate::counters::WS_RUNS, 1);
1268        crate::counters::add(&crate::counters::WS_BYTES, (i - start) as u64);
1269        Ok(next)
1270    }
1271
1272    fn position(&self) -> Position {
1273        self.position_of_index(self.index)
1274    }
1275
1276    fn peek_position(&self) -> Position {
1277        // Cap it at slice.len() just in case the most recent call was next()
1278        // and it returned the last byte.
1279        self.position_of_index(cmp::min(self.slice.len(), self.index + 1))
1280    }
1281
1282    fn byte_offset(&self) -> usize {
1283        self.index
1284    }
1285
1286    fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'a, 's, str>> {
1287        self.parse_str_bytes(scratch, true, as_str)
1288    }
1289
1290    fn parse_str_raw<'s>(
1291        &'s mut self,
1292        scratch: &'s mut Vec<u8>,
1293    ) -> Result<Reference<'a, 's, [u8]>> {
1294        self.parse_str_bytes(scratch, false, |_, bytes| Ok(bytes))
1295    }
1296
1297    fn ignore_str(&mut self) -> Result<()> {
1298        loop {
1299            self.skip_to_escape(true);
1300            if self.index == self.slice.len() {
1301                return error(self, ErrorCode::EofWhileParsingString);
1302            }
1303            match self.slice[self.index] {
1304                b'"' => {
1305                    self.index += 1;
1306                    return Ok(());
1307                }
1308                b'\\' => {
1309                    self.index += 1;
1310                    match ignore_escape(self) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(ignore_escape(self));
1311                }
1312                _ => {
1313                    return error(self, ErrorCode::ControlCharacterWhileParsingString);
1314                }
1315            }
1316        }
1317    }
1318
1319    #[inline]
1320    fn decode_hex_escape(&mut self) -> Result<u16> {
1321        match self.slice[self.index..] {
1322            [a, b, c, d, ..] => {
1323                self.index += 4;
1324                match decode_four_hex_digits(a, b, c, d) {
1325                    Some(val) => Ok(val),
1326                    None => error(self, ErrorCode::InvalidEscape),
1327                }
1328            }
1329            _ => {
1330                self.index = self.slice.len();
1331                error(self, ErrorCode::EofWhileParsingString)
1332            }
1333        }
1334    }
1335
1336    #[cfg(feature = "raw_value")]
1337    fn begin_raw_buffering(&mut self) {
1338        self.raw_buffering_start_index = self.index;
1339    }
1340
1341    #[cfg(feature = "raw_value")]
1342    fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
1343    where
1344        V: Visitor<'a>,
1345    {
1346        let raw = &self.slice[self.raw_buffering_start_index..self.index];
1347        let raw = match str::from_utf8(raw) {
1348            Ok(raw) => raw,
1349            Err(_) => return error(self, ErrorCode::InvalidUnicodeCodePoint),
1350        };
1351        visitor.visit_map(BorrowedRawDeserializer {
1352            raw_value: Some(raw),
1353        })
1354    }
1355
1356    const should_early_return_if_failed: bool = false;
1357
1358    #[inline]
1359    #[cold]
1360    fn set_failed(&mut self, _failed: &mut bool) {
1361        self.slice = &self.slice[..self.index];
1362    }
1363}
1364
1365//////////////////////////////////////////////////////////////////////////////
1366
1367impl<'a> StrRead<'a> {
1368    /// Create a JSON input source to read from a UTF-8 string.
1369    pub fn new(s: &'a str) -> Self {
1370        StrRead {
1371            delegate: SliceRead::new(s.as_bytes()),
1372            #[cfg(feature = "raw_value")]
1373            data: s,
1374        }
1375    }
1376}
1377
1378impl<'a> private::Sealed for StrRead<'a> {}
1379
1380impl<'a> Read<'a> for StrRead<'a> {
1381    #[inline]
1382    fn next(&mut self) -> Result<Option<u8>> {
1383        self.delegate.next()
1384    }
1385
1386    #[inline]
1387    fn peek(&mut self) -> Result<Option<u8>> {
1388        self.delegate.peek()
1389    }
1390
1391    #[inline]
1392    fn discard(&mut self) {
1393        self.delegate.discard();
1394    }
1395
1396    #[inline]
1397    fn skip_whitespace(&mut self) -> Result<Option<u8>> {
1398        self.delegate.skip_whitespace()
1399    }
1400
1401    #[inline]
1402    fn take_8_digits(&mut self) -> Option<u32> {
1403        self.delegate.take_8_digits()
1404    }
1405
1406    #[inline]
1407    fn position(&self) -> Position {
1408        self.delegate.position()
1409    }
1410
1411    fn peek_position(&self) -> Position {
1412        self.delegate.peek_position()
1413    }
1414
1415    fn byte_offset(&self) -> usize {
1416        self.delegate.byte_offset()
1417    }
1418
1419    fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'a, 's, str>> {
1420        self.delegate.parse_str_bytes(scratch, true, |_, bytes| {
1421            // The deserialization input came in as &str with a UTF-8 guarantee,
1422            // and the \u-escapes are checked along the way, so don't need to
1423            // check here.
1424            Ok(unsafe { str::from_utf8_unchecked(bytes) })
1425        })
1426    }
1427
1428    fn parse_str_raw<'s>(
1429        &'s mut self,
1430        scratch: &'s mut Vec<u8>,
1431    ) -> Result<Reference<'a, 's, [u8]>> {
1432        self.delegate.parse_str_raw(scratch)
1433    }
1434
1435    fn ignore_str(&mut self) -> Result<()> {
1436        self.delegate.ignore_str()
1437    }
1438
1439    fn decode_hex_escape(&mut self) -> Result<u16> {
1440        self.delegate.decode_hex_escape()
1441    }
1442
1443    #[cfg(feature = "raw_value")]
1444    fn begin_raw_buffering(&mut self) {
1445        self.delegate.begin_raw_buffering();
1446    }
1447
1448    #[cfg(feature = "raw_value")]
1449    fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
1450    where
1451        V: Visitor<'a>,
1452    {
1453        let raw = &self.data[self.delegate.raw_buffering_start_index..self.delegate.index];
1454        visitor.visit_map(BorrowedRawDeserializer {
1455            raw_value: Some(raw),
1456        })
1457    }
1458
1459    const should_early_return_if_failed: bool = false;
1460
1461    #[inline]
1462    #[cold]
1463    fn set_failed(&mut self, failed: &mut bool) {
1464        self.delegate.set_failed(failed);
1465    }
1466}
1467
1468//////////////////////////////////////////////////////////////////////////////
1469
1470impl<'de, R> private::Sealed for &mut R where R: Read<'de> {}
1471
1472impl<'de, R> Read<'de> for &mut R
1473where
1474    R: Read<'de>,
1475{
1476    fn next(&mut self) -> Result<Option<u8>> {
1477        R::next(self)
1478    }
1479
1480    fn peek(&mut self) -> Result<Option<u8>> {
1481        R::peek(self)
1482    }
1483
1484    fn discard(&mut self) {
1485        R::discard(self);
1486    }
1487
1488    fn position(&self) -> Position {
1489        R::position(self)
1490    }
1491
1492    fn peek_position(&self) -> Position {
1493        R::peek_position(self)
1494    }
1495
1496    fn byte_offset(&self) -> usize {
1497        R::byte_offset(self)
1498    }
1499
1500    fn parse_str<'s>(&'s mut self, scratch: &'s mut Vec<u8>) -> Result<Reference<'de, 's, str>> {
1501        R::parse_str(self, scratch)
1502    }
1503
1504    fn parse_str_raw<'s>(
1505        &'s mut self,
1506        scratch: &'s mut Vec<u8>,
1507    ) -> Result<Reference<'de, 's, [u8]>> {
1508        R::parse_str_raw(self, scratch)
1509    }
1510
1511    fn ignore_str(&mut self) -> Result<()> {
1512        R::ignore_str(self)
1513    }
1514
1515    fn decode_hex_escape(&mut self) -> Result<u16> {
1516        R::decode_hex_escape(self)
1517    }
1518
1519    #[cfg(feature = "raw_value")]
1520    fn begin_raw_buffering(&mut self) {
1521        R::begin_raw_buffering(self);
1522    }
1523
1524    #[cfg(feature = "raw_value")]
1525    fn end_raw_buffering<V>(&mut self, visitor: V) -> Result<V::Value>
1526    where
1527        V: Visitor<'de>,
1528    {
1529        R::end_raw_buffering(self, visitor)
1530    }
1531
1532    const should_early_return_if_failed: bool = R::should_early_return_if_failed;
1533
1534    fn set_failed(&mut self, failed: &mut bool) {
1535        R::set_failed(self, failed);
1536    }
1537}
1538
1539//////////////////////////////////////////////////////////////////////////////
1540
1541/// Marker for whether StreamDeserializer can implement FusedIterator.
1542pub trait Fused: private::Sealed {}
1543impl<'a> Fused for SliceRead<'a> {}
1544impl<'a> Fused for StrRead<'a> {}
1545
1546fn is_escape(ch: u8, including_control_characters: bool) -> bool {
1547    ch == b'"' || ch == b'\\' || (including_control_characters && ch < 0x20)
1548}
1549
1550fn next_or_eof<'de, R>(read: &mut R) -> Result<u8>
1551where
1552    R: ?Sized + Read<'de>,
1553{
1554    match match read.next() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.next()) {
1555        Some(b) => Ok(b),
1556        None => error(read, ErrorCode::EofWhileParsingString),
1557    }
1558}
1559
1560fn peek_or_eof<'de, R>(read: &mut R) -> Result<u8>
1561where
1562    R: ?Sized + Read<'de>,
1563{
1564    match match read.peek() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.peek()) {
1565        Some(b) => Ok(b),
1566        None => error(read, ErrorCode::EofWhileParsingString),
1567    }
1568}
1569
1570fn error<'de, R, T>(read: &R, reason: ErrorCode) -> Result<T>
1571where
1572    R: ?Sized + Read<'de>,
1573{
1574    let position = read.position();
1575    Err(Error::syntax(reason, position.line, position.column))
1576}
1577
1578fn as_str<'de, 's, R: Read<'de>>(read: &R, slice: &'s [u8]) -> Result<&'s str> {
1579    // PROBE (brick B12): this is the whole UTF-8 validation bill for
1580    // `from_slice`. `from_str` pays none of it, so the two together bound the
1581    // brick without any new code needing to be written.
1582    crate::counters::add(&crate::counters::UTF8_CALLS, 1);
1583    crate::counters::add(&crate::counters::UTF8_BYTES, slice.len() as u64);
1584    str::from_utf8(slice).or_else(|_| error(read, ErrorCode::InvalidUnicodeCodePoint))
1585}
1586
1587/// Parses a JSON escape sequence and appends it into the scratch space. Assumes
1588/// the previous byte read was a backslash.
1589fn parse_escape<'de, R: Read<'de>>(
1590    read: &mut R,
1591    validate: bool,
1592    scratch: &mut Vec<u8>,
1593) -> Result<()> {
1594    let ch = match next_or_eof(read) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(read));
1595
1596    match ch {
1597        b'"' => scratch.push(b'"'),
1598        b'\\' => scratch.push(b'\\'),
1599        b'/' => scratch.push(b'/'),
1600        b'b' => scratch.push(b'\x08'),
1601        b'f' => scratch.push(b'\x0c'),
1602        b'n' => scratch.push(b'\n'),
1603        b'r' => scratch.push(b'\r'),
1604        b't' => scratch.push(b'\t'),
1605        b'u' => return parse_unicode_escape(read, validate, scratch),
1606        _ => return error(read, ErrorCode::InvalidEscape),
1607    }
1608
1609    Ok(())
1610}
1611
1612/// Parses a JSON \u escape and appends it into the scratch space. Assumes `\u`
1613/// has just been read.
1614#[cold]
1615fn parse_unicode_escape<'de, R: Read<'de>>(
1616    read: &mut R,
1617    validate: bool,
1618    scratch: &mut Vec<u8>,
1619) -> Result<()> {
1620    let mut n = match read.decode_hex_escape() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.decode_hex_escape());
1621
1622    // Non-BMP characters are encoded as a sequence of two hex escapes,
1623    // representing UTF-16 surrogates. If deserializing a utf-8 string the
1624    // surrogates are required to be paired, whereas deserializing a byte string
1625    // accepts lone surrogates.
1626    if validate && n >= 0xDC00 && n <= 0xDFFF {
1627        // XXX: This is actually a trailing surrogate.
1628        return error(read, ErrorCode::LoneLeadingSurrogateInHexEscape);
1629    }
1630
1631    loop {
1632        if n < 0xD800 || n > 0xDBFF {
1633            // Every u16 outside of the surrogate ranges is guaranteed to be a
1634            // legal char.
1635            push_wtf8_codepoint(n as u32, scratch);
1636            return Ok(());
1637        }
1638
1639        // n is a leading surrogate, we now expect a trailing surrogate.
1640        let n1 = n;
1641
1642        if match peek_or_eof(read) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(peek_or_eof(read)) == b'\\' {
1643            read.discard();
1644        } else {
1645            return if validate {
1646                read.discard();
1647                error(read, ErrorCode::UnexpectedEndOfHexEscape)
1648            } else {
1649                push_wtf8_codepoint(n1 as u32, scratch);
1650                Ok(())
1651            };
1652        }
1653
1654        if match peek_or_eof(read) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(peek_or_eof(read)) == b'u' {
1655            read.discard();
1656        } else {
1657            return if validate {
1658                read.discard();
1659                error(read, ErrorCode::UnexpectedEndOfHexEscape)
1660            } else {
1661                push_wtf8_codepoint(n1 as u32, scratch);
1662                // The \ prior to this byte started an escape sequence, so we
1663                // need to parse that now. This recursive call does not blow the
1664                // stack on malicious input because the escape is not \u, so it
1665                // will be handled by one of the easy nonrecursive cases.
1666                parse_escape(read, validate, scratch)
1667            };
1668        }
1669
1670        let n2 = match read.decode_hex_escape() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(read.decode_hex_escape());
1671
1672        if n2 < 0xDC00 || n2 > 0xDFFF {
1673            if validate {
1674                return error(read, ErrorCode::LoneLeadingSurrogateInHexEscape);
1675            }
1676            push_wtf8_codepoint(n1 as u32, scratch);
1677            // If n2 is a leading surrogate, we need to restart.
1678            n = n2;
1679            continue;
1680        }
1681
1682        // This value is in range U+10000..=U+10FFFF, which is always a valid
1683        // codepoint.
1684        let n = ((((n1 - 0xD800) as u32) << 10) | (n2 - 0xDC00) as u32) + 0x1_0000;
1685        push_wtf8_codepoint(n, scratch);
1686        return Ok(());
1687    }
1688}
1689
1690/// Adds a WTF-8 codepoint to the end of the buffer. This is a more efficient
1691/// implementation of String::push. The codepoint may be a surrogate.
1692#[inline]
1693fn push_wtf8_codepoint(n: u32, scratch: &mut Vec<u8>) {
1694    if n < 0x80 {
1695        scratch.push(n as u8);
1696        return;
1697    }
1698
1699    scratch.reserve(4);
1700
1701    // SAFETY: After the `reserve` call, `scratch` has at least 4 bytes of
1702    // allocated but uninitialized memory after its last initialized byte, which
1703    // is where `ptr` points. All reachable match arms write `encoded_len` bytes
1704    // to that region and update the length accordingly, and `encoded_len` is
1705    // always <= 4.
1706    unsafe {
1707        let ptr = scratch.as_mut_ptr().add(scratch.len());
1708
1709        let encoded_len = match n {
1710            0..=0x7F => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1711            0x80..=0x7FF => {
1712                ptr.write(((n >> 6) & 0b0001_1111) as u8 | 0b1100_0000);
1713                2
1714            }
1715            0x800..=0xFFFF => {
1716                ptr.write(((n >> 12) & 0b0000_1111) as u8 | 0b1110_0000);
1717                ptr.add(1)
1718                    .write(((n >> 6) & 0b0011_1111) as u8 | 0b1000_0000);
1719                3
1720            }
1721            0x1_0000..=0x10_FFFF => {
1722                ptr.write(((n >> 18) & 0b0000_0111) as u8 | 0b1111_0000);
1723                ptr.add(1)
1724                    .write(((n >> 12) & 0b0011_1111) as u8 | 0b1000_0000);
1725                ptr.add(2)
1726                    .write(((n >> 6) & 0b0011_1111) as u8 | 0b1000_0000);
1727                4
1728            }
1729            0x11_0000.. => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1730        };
1731        ptr.add(encoded_len - 1)
1732            .write((n & 0b0011_1111) as u8 | 0b1000_0000);
1733
1734        scratch.set_len(scratch.len() + encoded_len);
1735    }
1736}
1737
1738/// Parses a JSON escape sequence and discards the value. Assumes the previous
1739/// byte read was a backslash.
1740fn ignore_escape<'de, R>(read: &mut R) -> Result<()>
1741where
1742    R: ?Sized + Read<'de>,
1743{
1744    let ch = match next_or_eof(read) {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
}tri!(next_or_eof(read));
1745
1746    match ch {
1747        b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't' => {}
1748        b'u' => {
1749            // At this point we don't care if the codepoint is valid. We just
1750            // want to consume it. We don't actually know what is valid or not
1751            // at this point, because that depends on if this string will
1752            // ultimately be parsed into a string or a byte buffer in the "real"
1753            // parse.
1754
1755            match read.decode_hex_escape() {
    core::result::Result::Ok(val) => val,
    core::result::Result::Err(err) => return core::result::Result::Err(err),
};tri!(read.decode_hex_escape());
1756        }
1757        _ => {
1758            return error(read, ErrorCode::InvalidEscape);
1759        }
1760    }
1761
1762    Ok(())
1763}
1764
1765const fn decode_hex_val_slow(val: u8) -> Option<u8> {
1766    match val {
1767        b'0'..=b'9' => Some(val - b'0'),
1768        b'A'..=b'F' => Some(val - b'A' + 10),
1769        b'a'..=b'f' => Some(val - b'a' + 10),
1770        _ => None,
1771    }
1772}
1773
1774const fn build_hex_table(shift: usize) -> [i16; 256] {
1775    let mut table = [0; 256];
1776    let mut ch = 0;
1777    while ch < 256 {
1778        table[ch] = match decode_hex_val_slow(ch as u8) {
1779            Some(val) => (val as i16) << shift,
1780            None => -1,
1781        };
1782        ch += 1;
1783    }
1784    table
1785}
1786
1787static HEX0: [i16; 256] = build_hex_table(0);
1788static HEX1: [i16; 256] = build_hex_table(4);
1789
1790fn decode_four_hex_digits(a: u8, b: u8, c: u8, d: u8) -> Option<u16> {
1791    let a = HEX1[a as usize] as i32;
1792    let b = HEX0[b as usize] as i32;
1793    let c = HEX1[c as usize] as i32;
1794    let d = HEX0[d as usize] as i32;
1795
1796    let codepoint = ((a | b) << 8) | c | d;
1797
1798    // A single sign bit check.
1799    if codepoint >= 0 {
1800        Some(codepoint as u16)
1801    } else {
1802        None
1803    }
1804}
1805
1806#[cfg(test)]
1807mod ws_tests {
1808    use super::{is_ws, scan_ws, scan_ws_scalar};
1809
1810    /// Index only, for the comparisons below.
1811    fn wide(s: &[u8], from: usize) -> usize {
1812        scan_ws(s, from).0
1813    }
1814    fn scalar(s: &[u8], from: usize) -> usize {
1815        scan_ws_scalar(s, from).0
1816    }
1817
1818    /// The byte each scanner reports must be the byte at the index it reports.
1819    /// Returning a stale or re-loaded byte here would be invisible to an
1820    /// index-only comparison and would corrupt every token boundary.
1821    fn agree(s: &[u8], from: usize) {
1822        let (wi, wb) = scan_ws(s, from);
1823        let (si, sb) = scan_ws_scalar(s, from);
1824        assert_eq!(wi, si, "index disagreement at {from} in {s:?}");
1825        assert_eq!(wb, sb, "byte disagreement at {from} in {s:?}");
1826        assert_eq!(wb, s.get(wi).copied(), "byte is not the one at the index");
1827    }
1828    use alloc::vec;
1829    use alloc::vec::Vec;
1830
1831    /// The gate the house requires of any wide kernel: it must agree with its
1832    /// scalar twin on every byte value at every offset, not merely on a corpus.
1833    #[test]
1834    fn wide_matches_scalar_for_every_byte_at_every_offset() {
1835        // A window long enough that the wide loop runs at least twice, so a
1836        // byte can be placed before, inside and after the first chunk.
1837        for pos in 0..24usize {
1838            for b in 0..=255u8 {
1839                let mut buf = [b' '; 24];
1840                buf[pos] = b;
1841                for from in 0..24usize {
1842                    let want = scalar(&buf, from);
1843                    let got = wide(&buf, from);
1844                    assert_eq!(
1845                        got, want,
1846                        "byte {b:#04x} at {pos}, scanning from {from}: wide said {got}, scalar {want}"
1847                    );
1848                    // ... and the byte reported must be the byte at that index.
1849                    agree(&buf, from);
1850                }
1851            }
1852        }
1853    }
1854
1855    /// Every length from empty to past two chunks, all-whitespace and none.
1856    #[test]
1857    fn wide_matches_scalar_at_every_length() {
1858        for len in 0..40usize {
1859            for fill in [b' ', b'\t', b'\n', b'\r', b'x', 0x0b, 0x0c, 0x00] {
1860                let buf = vec![fill; len];
1861                for from in 0..=len {
1862                    assert_eq!(
1863                        wide(&buf, from),
1864                        scalar(&buf, from),
1865                        "fill {fill:#04x}, len {len}, from {from}"
1866                    );
1867                    agree(&buf, from);
1868                }
1869            }
1870        }
1871    }
1872
1873    /// Mixed content, including the bytes a range test would wrongly skip.
1874    #[test]
1875    fn wide_matches_scalar_on_mixed_content() {
1876        let mut seed = 0x1234_5678_9abc_def0u64;
1877        let alphabet = [
1878            b' ', b' ', b' ', b'\t', b'\n', b'\r', // whitespace, over-represented
1879            0x0b, 0x0c, 0x00, 0x1f, // NOT whitespace, and <= 0x20: the trap
1880            b'{', b'}', b'"', b'1', b'a', 0xff,
1881        ];
1882        for _ in 0..2000 {
1883            let mut buf = Vec::new();
1884            for _ in 0..64 {
1885                seed = seed
1886                    .wrapping_mul(6364136223846793005)
1887                    .wrapping_add(1442695040888963407);
1888                buf.push(alphabet[(seed >> 33) as usize % alphabet.len()]);
1889            }
1890            for from in 0..buf.len() {
1891                assert_eq!(
1892                    wide(&buf, from),
1893                    scalar(&buf, from),
1894                    "buf {buf:?} from {from}"
1895                );
1896                agree(&buf, from);
1897            }
1898        }
1899    }
1900
1901    /// The control chars a `b <= 0x20` or `0x09..=0x0d` test would wrongly
1902    /// treat as whitespace. Skipping them would silently accept invalid JSON.
1903    #[test]
1904    fn vertical_tab_and_form_feed_are_not_whitespace() {
1905        for b in [0x0b_u8, 0x0c, 0x00, 0x01, 0x1f, 0x21] {
1906            assert!(!is_ws(b), "{b:#04x} must not be skippable whitespace");
1907            let mut buf = [b' '; 16];
1908            buf[10] = b;
1909            assert_eq!(wide(&buf, 0), 10);
1910            assert_eq!(scalar(&buf, 0), 10);
1911        }
1912    }
1913}
1914
1915#[cfg(test)]
1916mod digit_tests {
1917    use super::{all_8_digits, eight_digits_value};
1918    use alloc::format;
1919
1920    /// What the SWAR fold must agree with, computed the obvious way.
1921    fn scalar_value(s: &[u8; 8]) -> u32 {
1922        s.iter()
1923            .fold(0u32, |acc, &c| acc * 10 + u32::from(c - b'0'))
1924    }
1925
1926    #[test]
1927    fn accepts_only_all_digit_chunks() {
1928        let digits = *b"12345678";
1929        assert!(all_8_digits(u64::from_le_bytes(digits)));
1930        // A single non-digit anywhere must reject the chunk. Byte values are
1931        // exhaustive, positions are exhaustive.
1932        for pos in 0..8 {
1933            for b in 0..=255u8 {
1934                let mut buf = digits;
1935                buf[pos] = b;
1936                let want = b.is_ascii_digit();
1937                assert_eq!(
1938                    all_8_digits(u64::from_le_bytes(buf)),
1939                    want,
1940                    "byte {b:#04x} at {pos}"
1941                );
1942            }
1943        }
1944    }
1945
1946    #[test]
1947    fn value_matches_scalar_on_every_boundary_and_at_random() {
1948        for s in [
1949            b"00000000",
1950            b"00000001",
1951            b"10000000",
1952            b"99999999",
1953            b"12345678",
1954            b"87654321",
1955            b"09090909",
1956            b"90909090",
1957        ] {
1958            let chunk = u64::from_le_bytes(*s);
1959            assert!(all_8_digits(chunk));
1960            assert_eq!(eight_digits_value(chunk), scalar_value(s), "{:?}", s);
1961        }
1962        // Every digit in every position, the rest zeroes: catches a fold that
1963        // weights a position wrongly.
1964        for pos in 0..8 {
1965            for d in b'0'..=b'9' {
1966                let mut buf = *b"00000000";
1967                buf[pos] = d;
1968                let chunk = u64::from_le_bytes(buf);
1969                assert_eq!(
1970                    eight_digits_value(chunk),
1971                    scalar_value(&buf),
1972                    "digit {} at {pos}",
1973                    d as char
1974                );
1975            }
1976        }
1977        let mut seed = 0x9e37_79b9_7f4a_7c15u64;
1978        for _ in 0..20_000 {
1979            seed = seed
1980                .wrapping_mul(6364136223846793005)
1981                .wrapping_add(1442695040888963407);
1982            let n = (seed >> 32) % 100_000_000;
1983            let s = format!("{n:08}");
1984            let buf = <[u8; 8]>::try_from(s.as_bytes()).unwrap();
1985            let chunk = u64::from_le_bytes(buf);
1986            assert!(all_8_digits(chunk), "{s}");
1987            assert_eq!(eight_digits_value(chunk), n as u32, "{s}");
1988        }
1989    }
1990}