Skip to main content

oneyaml_syntax/
decode.rs

1//! Input decoding (YAML 1.2.2 §5.2): bytes in UTF-8, UTF-16 LE/BE, or
2//! UTF-32 LE/BE to the UTF-8 text the parser reads.
3//!
4//! In-house and safe: `std`'s `str::from_utf8`, `char::decode_utf16`,
5//! and `char::from_u32` do the work; there are no dependencies. One forward
6//! pass, O(n), no recursion.
7//!
8//! # Round trip
9//!
10//! A byte order mark is **kept** in the text as U+FEFF, where the parser
11//! reads it as [`TriviaKind::ByteOrderMark`](crate::event::TriviaKind). UTF-16
12//! and UTF-32 have exactly one encoding per character, so for valid input
13//! `encode(decoded.text(), decoded.encoding())` reproduces the input bytes
14//! exactly: the [`Encoding`] is all an emitter needs to record.
15//!
16//! # Invalid input
17//!
18//! Decoding never fails and never panics. Each maximal run of invalid bytes
19//! becomes one U+FFFD in the text and one error diagnostic
20//! (`INVALID_UTF8`/`INVALID_UTF16`/`INVALID_UTF32`) whose label covers the
21//! U+FFFD in the text. The original bytes of each run are kept, compactly
22//! ([`InvalidRuns`]: offsets, and one pool of the invalid bytes), so
23//! [`Decoded::reencode`] still reproduces the input exactly. **The diagnostics are not part of the event stream**: callers
24//! must check [`Decoded::diagnostics`] (a strict caller rejects the input
25//! when it is not empty).
26
27use std::borrow::Cow;
28
29use oneyaml_core::code::{INTERNAL_ERROR, INVALID_UTF8, INVALID_UTF16, INVALID_UTF32};
30use oneyaml_core::{Code, Diagnostic, Label, Span};
31
32pub use oneyaml_core::Encoding;
33
34/// The decoder's view of an encoding. `Encoding` is vocabulary in
35/// `oneyaml-core` and `#[non_exhaustive]` there; detection yields a
36/// `Unit` directly, so decoding never meets a variant it cannot read.
37#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
38enum Unit {
39    Utf8,
40    Utf16 { big: bool },
41    Utf32 { big: bool },
42}
43
44/// The encoding a unit decodes (total).
45const fn encoding_of(unit: Unit) -> Encoding {
46    match unit {
47        Unit::Utf8 => Encoding::Utf8,
48        Unit::Utf16 { big: false } => Encoding::Utf16Le,
49        Unit::Utf16 { big: true } => Encoding::Utf16Be,
50        Unit::Utf32 { big: false } => Encoding::Utf32Le,
51        Unit::Utf32 { big: true } => Encoding::Utf32Be,
52    }
53}
54
55/// The unit of a caller's `Encoding`. Never a panic: a variant added to `-core`
56/// without codec support is an internal error (`Y9000`), never a silent
57/// read or write as another encoding.
58fn unit(encoding: Encoding) -> Result<Unit, Box<Diagnostic>> {
59    match encoding {
60        Encoding::Utf8 => Ok(Unit::Utf8),
61        Encoding::Utf16Le => Ok(Unit::Utf16 { big: false }),
62        Encoding::Utf16Be => Ok(Unit::Utf16 { big: true }),
63        Encoding::Utf32Le => Ok(Unit::Utf32 { big: false }),
64        Encoding::Utf32Be => Ok(Unit::Utf32 { big: true }),
65        _ => {
66            debug_assert!(false, "no codec for {encoding:?}");
67            Err(Box::new(Diagnostic::error(
68                INTERNAL_ERROR,
69                format!("no codec for the encoding {}", encoding.name()),
70                Label::new(Span::empty(0), ""),
71            )))
72        }
73    }
74}
75
76/// The unit per the table in YAML 1.2.2 §5.2: a byte order mark, or else
77/// the pattern of null bytes around an ASCII first character; UTF-8
78/// otherwise.
79const fn detect_unit(bytes: &[u8]) -> Unit {
80    match bytes {
81        [0, 0, 0xFE, 0xFF, ..] | [0, 0, 0, _, ..] => Unit::Utf32 { big: true },
82        [0xFF, 0xFE, 0, 0, ..] | [_, 0, 0, 0, ..] => Unit::Utf32 { big: false },
83        [0xFE, 0xFF, ..] | [0, _, ..] => Unit::Utf16 { big: true },
84        [0xFF, 0xFE, ..] | [_, 0, ..] => Unit::Utf16 { big: false },
85        _ => Unit::Utf8,
86    }
87}
88
89/// Detects the encoding of `bytes` from their first bytes (YAML 1.2.2
90/// §5.2): a byte order mark, or else the pattern of null bytes around an
91/// ASCII first character; UTF-8 otherwise. Detection lives here, not in
92/// `oneyaml-core`, which only names encodings.
93#[must_use]
94pub const fn detect_encoding(bytes: &[u8]) -> Encoding {
95    encoding_of(detect_unit(bytes))
96}
97
98/// An encoding this decoder can write: detected ([`Decoded::encoder`]),
99/// or checked from an [`Encoding`] ([`Encoder::new`]). Encoding with it
100/// never fails.
101#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
102pub struct Encoder(Unit);
103
104impl Default for Encoder {
105    /// UTF-8.
106    fn default() -> Self {
107        Self(Unit::Utf8)
108    }
109}
110
111impl Encoder {
112    /// The encoder for `encoding`.
113    ///
114    /// # Errors
115    ///
116    /// An internal error (`Y9000`) for an encoding this version cannot
117    /// write (none today: `Encoding` is `#[non_exhaustive]`).
118    pub fn new(encoding: Encoding) -> Result<Self, Box<Diagnostic>> {
119        unit(encoding).map(Self)
120    }
121
122    /// The encoding written.
123    #[must_use]
124    pub const fn encoding(self) -> Encoding {
125        encoding_of(self.0)
126    }
127
128    /// Encodes `text` (byte order marks included).
129    #[must_use]
130    pub fn encode(self, text: &str) -> Vec<u8> {
131        let mut out = Vec::new();
132        encode_into(&mut out, text, self.0);
133        out
134    }
135}
136
137/// The code of an encoding's decoding error.
138const fn error_code(unit: Unit) -> Code {
139    match unit {
140        Unit::Utf8 => INVALID_UTF8,
141        Unit::Utf16 { .. } => INVALID_UTF16,
142        Unit::Utf32 { .. } => INVALID_UTF32,
143    }
144}
145
146/// A run of invalid input bytes, replaced by one U+FFFD in the text: a
147/// view into [`InvalidRuns`].
148#[non_exhaustive]
149#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
150pub struct InvalidRun<'r> {
151    /// The U+FFFD in the decoded text (UTF-8 byte range).
152    pub text: Span,
153    /// The byte range of the run in the input.
154    pub input: Span,
155    /// The original bytes of the run.
156    pub bytes: &'r [u8],
157}
158
159/// The length of U+FFFD in UTF-8: every run's text.
160const REPLACEMENT_LEN: usize = char::REPLACEMENT_CHARACTER.len_utf8();
161
162/// The invalid runs of decoded input, in input order, stored compactly:
163/// per run, its text start, its input start, and where
164/// its bytes start in one pool of all the invalid bytes (three `u32`s when
165/// the input is small enough, else three `usize`s); no allocation per run.
166/// The pool keeps every invalid byte, so the input is still reproduced
167/// exactly.
168///
169/// Equality, hashing, and `Debug` are over the runs, never the
170/// storage form.
171#[derive(Clone, Default)]
172pub struct InvalidRuns {
173    recs: Recs,
174    pool: Vec<u8>,
175}
176
177impl PartialEq for InvalidRuns {
178    fn eq(&self, other: &Self) -> bool {
179        self.len() == other.len() && self.iter().eq(other.iter())
180    }
181}
182
183impl Eq for InvalidRuns {}
184
185impl core::hash::Hash for InvalidRuns {
186    fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
187        state.write_usize(self.len());
188        for run in self {
189            run.hash(state);
190        }
191    }
192}
193
194impl core::fmt::Debug for InvalidRuns {
195    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
196        f.debug_list().entries(self.iter()).finish()
197    }
198}
199
200/// `[text start, input start, pool start]` per run.
201#[derive(Clone, Debug)]
202enum Recs {
203    Narrow(Vec<[u32; 3]>),
204    Wide(Vec<[usize; 3]>),
205}
206
207impl Default for Recs {
208    fn default() -> Self {
209        Self::Narrow(Vec::new())
210    }
211}
212
213impl Recs {
214    fn len(&self) -> usize {
215        match self {
216            Self::Narrow(v) => v.len(),
217            Self::Wide(v) => v.len(),
218        }
219    }
220
221    fn get(&self, i: usize) -> Option<[usize; 3]> {
222        match self {
223            Self::Narrow(v) => v.get(i).map(|r| r.map(|x| x as usize)),
224            Self::Wide(v) => v.get(i).copied(),
225        }
226    }
227
228    /// Appends a record. Narrow records that cannot hold it become wide
229    /// first (never with the records
230    /// [`InvalidRuns::for_input`] chose, and never a dropped run).
231    fn push(&mut self, rec: [usize; 3]) {
232        if let Self::Narrow(v) = self {
233            if let [Ok(a), Ok(b), Ok(c)] = rec.map(u32::try_from) {
234                v.push([a, b, c]);
235                return;
236            }
237            *self = Self::Wide(v.iter().map(|r| r.map(|x| x as usize)).collect());
238        }
239        if let Self::Wide(v) = self {
240            v.push(rec);
241        }
242    }
243
244    fn shrink_to_fit(&mut self) {
245        match self {
246            Self::Narrow(v) => v.shrink_to_fit(),
247            Self::Wide(v) => v.shrink_to_fit(),
248        }
249    }
250
251    /// The first record whose text start is at least `at`.
252    fn first_at(&self, at: usize) -> usize {
253        match self {
254            Self::Narrow(v) => v.partition_point(|r| (r[0] as usize) < at),
255            Self::Wide(v) => v.partition_point(|r| r[0] < at),
256        }
257    }
258
259    fn heap_bytes(&self) -> usize {
260        match self {
261            Self::Narrow(v) => v.capacity().saturating_mul(size_of::<[u32; 3]>()),
262            Self::Wide(v) => v.capacity().saturating_mul(size_of::<[usize; 3]>()),
263        }
264    }
265}
266
267impl InvalidRuns {
268    /// Runs of input of `len` bytes: narrow records when every offset
269    /// (the text's included: at most three bytes per input byte) fits a
270    /// `u32`.
271    fn for_input(len: usize) -> Self {
272        let narrow = len
273            .checked_mul(REPLACEMENT_LEN)
274            .is_some_and(|x| u32::try_from(x).is_ok());
275        let recs = if narrow {
276            Recs::Narrow(Vec::new())
277        } else {
278            Recs::Wide(Vec::new())
279        };
280        Self {
281            recs,
282            pool: Vec::new(),
283        }
284    }
285
286    /// How many runs.
287    #[must_use]
288    pub fn len(&self) -> usize {
289        self.recs.len()
290    }
291
292    /// `true` for valid input.
293    #[must_use]
294    pub fn is_empty(&self) -> bool {
295        self.len() == 0
296    }
297
298    /// Run `i`, in input order. O(1).
299    #[must_use]
300    pub fn get(&self, i: usize) -> Option<InvalidRun<'_>> {
301        let [text, input, from] = self.recs.get(i)?;
302        let to = match self.recs.get(i.checked_add(1)?) {
303            Some([_, _, next]) => next,
304            None => self.pool.len(),
305        };
306        let bytes = self.pool.get(from..to)?;
307        Some(InvalidRun {
308            text: Span::new(text, text.saturating_add(REPLACEMENT_LEN)),
309            input: Span::new(input, input.saturating_add(bytes.len())),
310            bytes,
311        })
312    }
313
314    /// The first run whose text starts at or after the text offset `at`;
315    /// `len()` if none does. O(log n).
316    #[must_use]
317    pub fn first_at(&self, at: usize) -> usize {
318        self.recs.first_at(at)
319    }
320
321    /// The runs, in input order.
322    #[must_use]
323    pub fn iter(&self) -> InvalidRunsIter<'_> {
324        InvalidRunsIter {
325            runs: self,
326            at: 0,
327            end: self.len(),
328        }
329    }
330
331    /// The heap bytes held: the records and the pool of invalid bytes
332    /// (for the memory accounting).
333    #[must_use]
334    #[doc(hidden)]
335    pub fn heap_bytes(&self) -> usize {
336        self.recs.heap_bytes().saturating_add(self.pool.capacity())
337    }
338
339    /// Appends a run whose U+FFFD starts at `text`.
340    fn push(&mut self, text: usize, input: usize, bytes: &[u8]) {
341        self.recs.push([text, input, self.pool.len()]);
342        self.pool.extend_from_slice(bytes);
343    }
344
345    /// The last run's input and text ends.
346    fn last_ends(&self) -> Option<(usize, usize)> {
347        let r = self.get(self.len().checked_sub(1)?)?;
348        Some((r.input.end, r.text.end))
349    }
350
351    fn shrink_to_fit(&mut self) {
352        self.recs.shrink_to_fit();
353        self.pool.shrink_to_fit();
354    }
355}
356
357impl<'r> IntoIterator for &'r InvalidRuns {
358    type Item = InvalidRun<'r>;
359    type IntoIter = InvalidRunsIter<'r>;
360
361    fn into_iter(self) -> Self::IntoIter {
362        self.iter()
363    }
364}
365
366/// The iterator of [`InvalidRuns::iter`]: O(1) per step, `nth` included.
367#[derive(Clone, Debug)]
368pub struct InvalidRunsIter<'r> {
369    runs: &'r InvalidRuns,
370    at: usize,
371    end: usize,
372}
373
374impl<'r> Iterator for InvalidRunsIter<'r> {
375    type Item = InvalidRun<'r>;
376
377    fn next(&mut self) -> Option<Self::Item> {
378        if self.at >= self.end {
379            return None;
380        }
381        let r = self.runs.get(self.at);
382        self.at = self.at.saturating_add(1);
383        r
384    }
385
386    fn nth(&mut self, n: usize) -> Option<Self::Item> {
387        self.at = self.at.saturating_add(n).min(self.end);
388        self.next()
389    }
390
391    fn size_hint(&self) -> (usize, Option<usize>) {
392        let n = self.end.saturating_sub(self.at);
393        (n, Some(n))
394    }
395}
396
397impl DoubleEndedIterator for InvalidRunsIter<'_> {
398    fn next_back(&mut self) -> Option<Self::Item> {
399        if self.at >= self.end {
400            return None;
401        }
402        self.end = self.end.saturating_sub(1);
403        self.runs.get(self.end)
404    }
405}
406
407impl ExactSizeIterator for InvalidRunsIter<'_> {}
408
409/// Decoded input: the text, how it was encoded, and any decoding problems.
410#[derive(Clone, Debug, PartialEq, Eq)]
411pub struct Decoded<'a> {
412    text: Cow<'a, str>,
413    unit: Unit,
414    invalid: InvalidRuns,
415    diagnostics: Vec<Diagnostic>,
416    /// Runs without a built diagnostic (past the keep budget).
417    suppressed: usize,
418}
419
420impl<'a> Decoded<'a> {
421    /// The decoded text, byte order marks included. Borrowed from the input
422    /// for valid UTF-8.
423    #[must_use]
424    pub fn text(&self) -> &str {
425        &self.text
426    }
427
428    /// Consumes `self`, returning the text.
429    #[must_use]
430    pub fn into_text(self) -> Cow<'a, str> {
431        self.text
432    }
433
434    /// The detected encoding.
435    #[must_use]
436    pub const fn encoding(&self) -> Encoding {
437        encoding_of(self.unit)
438    }
439
440    /// The encoder that writes text back in the detected encoding.
441    #[must_use]
442    pub const fn encoder(&self) -> Encoder {
443        Encoder(self.unit)
444    }
445
446    /// Whether the input starts with a byte order mark.
447    #[must_use]
448    #[doc(hidden)]
449    pub fn has_bom(&self) -> bool {
450        self.text.starts_with('\u{feff}')
451    }
452
453    /// The invalid runs, in input order.
454    #[must_use]
455    pub fn invalid_runs(&self) -> &InvalidRuns {
456        &self.invalid
457    }
458
459    /// One error diagnostic per invalid run, in order; empty for valid
460    /// input. With [`decode_with_limit`], those of the first runs only
461    /// (see [`Decoded::suppressed_diagnostics`]).
462    #[must_use]
463    pub fn diagnostics(&self) -> &[Diagnostic] {
464        &self.diagnostics
465    }
466
467    /// How many invalid runs have no built diagnostic (past the
468    /// budget of [`decode_with_limit`]); [`Decoded::invalid_runs`] is
469    /// always complete, and [`run_diagnostic`] builds any run's.
470    #[must_use]
471    pub const fn suppressed_diagnostics(&self) -> usize {
472        self.suppressed
473    }
474
475    /// `true` when the input had no decoding error.
476    #[must_use]
477    pub fn is_valid(&self) -> bool {
478        self.invalid.is_empty()
479    }
480
481    /// Takes `self` apart (no copy of the runs): the built
482    /// diagnostics, the invalid runs, the encoder, and the text.
483    #[must_use]
484    pub fn into_parts(self) -> (Vec<Diagnostic>, InvalidRuns, Encoder, Cow<'a, str>) {
485        (
486            self.diagnostics,
487            self.invalid,
488            Encoder(self.unit),
489            self.text,
490        )
491    }
492
493    /// Re-encodes the unedited text, restoring invalid runs from their
494    /// original bytes: always equal to the input.
495    #[must_use]
496    pub fn reencode(&self) -> Vec<u8> {
497        let mut out = Vec::with_capacity(self.text.len());
498        let mut at = 0usize;
499        for run in &self.invalid {
500            let before = self.text.get(at..run.text.start).unwrap_or("");
501            encode_into(&mut out, before, self.unit);
502            out.extend_from_slice(run.bytes);
503            at = run.text.end;
504        }
505        encode_into(&mut out, self.text.get(at..).unwrap_or(""), self.unit);
506        out
507    }
508}
509
510/// Decodes `bytes`, detecting the encoding with [`detect_encoding`].
511/// Never fails; see the module docs for invalid input.
512#[must_use]
513pub fn decode(bytes: &[u8]) -> Decoded<'_> {
514    decode_with_limit(bytes, usize::MAX)
515}
516
517/// [`decode`], building the diagnostics of the first `keep` invalid runs
518/// only (`Limits::max_diagnostics`); the others are
519/// counted ([`Decoded::suppressed_diagnostics`]). The runs themselves are
520/// always all kept, and [`run_diagnostic`] builds any run's
521/// diagnostic exactly as decoding would.
522#[must_use]
523pub fn decode_with_limit(bytes: &[u8], keep: usize) -> Decoded<'_> {
524    let unit = detect_unit(bytes);
525    let mut d = Decoder {
526        text: String::new(),
527        unit,
528        invalid: InvalidRuns::for_input(bytes.len()),
529        diagnostics: Vec::new(),
530        keep,
531        max_runs: usize::MAX,
532        full: false,
533    };
534    match unit {
535        Unit::Utf8 => {
536            if let Ok(text) = core::str::from_utf8(bytes) {
537                return Decoded {
538                    text: Cow::Borrowed(text),
539                    unit,
540                    invalid: InvalidRuns::default(),
541                    diagnostics: Vec::new(),
542                    suppressed: 0,
543                };
544            }
545            d.utf8(bytes);
546        }
547        Unit::Utf16 { big } => d.utf16(bytes, big),
548        Unit::Utf32 { big } => d.utf32(bytes, big),
549    }
550    let suppressed = d.invalid.len().saturating_sub(d.diagnostics.len());
551    Decoded {
552        text: Cow::Owned(d.text),
553        unit,
554        invalid: d.invalid,
555        diagnostics: d.diagnostics,
556        suppressed,
557    }
558}
559
560/// The first decoding error of `bytes`, if any, exactly as [`decode`]
561/// reports it, with the text decoded up to it: decoding stops once its
562/// run is complete (a reader that fails at the first
563/// error holds nothing more).
564#[must_use]
565pub fn first_error(bytes: &[u8]) -> Option<(Diagnostic, String)> {
566    let unit = detect_unit(bytes);
567    if matches!(unit, Unit::Utf8) && core::str::from_utf8(bytes).is_ok() {
568        return None;
569    }
570    let mut d = Decoder {
571        text: String::new(),
572        unit,
573        invalid: InvalidRuns::for_input(bytes.len()),
574        diagnostics: Vec::new(),
575        keep: 1,
576        max_runs: 1,
577        full: false,
578    };
579    match unit {
580        Unit::Utf8 => d.utf8(bytes),
581        Unit::Utf16 { big } => d.utf16(bytes, big),
582        Unit::Utf32 { big } => d.utf32(bytes, big),
583    }
584    let text = d.text;
585    d.diagnostics.into_iter().next().map(|e| (e, text))
586}
587
588/// [`decode`] for a strict reader: one pass. Valid input
589/// is decoded as [`decode`] decodes it (borrowed for valid UTF-8, no
590/// copy); at the first invalid run decoding stops, as [`first_error`]
591/// does, and that error is returned with the text decoded up to it.
592///
593/// # Errors
594///
595/// The first decoding error, exactly as [`first_error`] reports it.
596pub fn decode_strict(bytes: &[u8]) -> Result<Decoded<'_>, Box<(Diagnostic, String)>> {
597    let unit = detect_unit(bytes);
598    if matches!(unit, Unit::Utf8) {
599        if let Ok(text) = core::str::from_utf8(bytes) {
600            return Ok(Decoded {
601                text: Cow::Borrowed(text),
602                unit,
603                invalid: InvalidRuns::default(),
604                diagnostics: Vec::new(),
605                suppressed: 0,
606            });
607        }
608    }
609    let mut d = Decoder {
610        text: String::new(),
611        unit,
612        invalid: InvalidRuns::for_input(bytes.len()),
613        diagnostics: Vec::new(),
614        keep: 1,
615        max_runs: 1,
616        full: false,
617    };
618    match unit {
619        Unit::Utf8 => d.utf8(bytes),
620        Unit::Utf16 { big } => d.utf16(bytes, big),
621        Unit::Utf32 { big } => d.utf32(bytes, big),
622    }
623    match d.diagnostics.into_iter().next() {
624        Some(e) => Err(Box::new((e, d.text))),
625        None => Ok(Decoded {
626            text: Cow::Owned(d.text),
627            unit,
628            invalid: d.invalid,
629            diagnostics: Vec::new(),
630            suppressed: 0,
631        }),
632    }
633}
634
635/// The diagnostic decoding reports for `run` of input decoded as
636/// `encoding` (the same as [`Decoded::diagnostics`] holds for it).
637#[must_use]
638pub fn run_diagnostic(encoding: Encoding, run: &InvalidRun<'_>) -> Diagnostic {
639    let unit = match encoding {
640        Encoding::Utf16Le => Unit::Utf16 { big: false },
641        Encoding::Utf16Be => Unit::Utf16 { big: true },
642        Encoding::Utf32Le => Unit::Utf32 { big: false },
643        Encoding::Utf32Be => Unit::Utf32 { big: true },
644        _ => Unit::Utf8,
645    };
646    diagnostic_of(unit, run)
647}
648
649fn diagnostic_of(unit: Unit, run: &InvalidRun<'_>) -> Diagnostic {
650    let name = encoding_of(unit).name();
651    let message = format!(
652        "invalid {name} at input bytes {}..{}",
653        run.input.start, run.input.end
654    );
655    Diagnostic::error(
656        error_code(unit),
657        message,
658        Label::new(run.text, "replaced with U+FFFD"),
659    )
660    .with_help(format!(
661        "the input was read as {name} (YAML 1.2.2 §5.2: detected from its first bytes); \
662         save it as valid {name} or UTF-8"
663    ))
664}
665
666/// Encodes `text` (byte order marks included) in `encoding`.
667///
668/// # Errors
669///
670/// As [`Encoder::new`]: an internal error (`Y9000`) for an encoding this
671/// version cannot write, never bytes in another encoding.
672pub fn encode(text: &str, encoding: Encoding) -> Result<Vec<u8>, Box<Diagnostic>> {
673    Encoder::new(encoding).map(|e| e.encode(text))
674}
675
676fn encode_into(out: &mut Vec<u8>, text: &str, unit: Unit) {
677    match unit {
678        Unit::Utf8 => out.extend_from_slice(text.as_bytes()),
679        Unit::Utf16 { big: false } => text
680            .encode_utf16()
681            .for_each(|u| out.extend_from_slice(&u.to_le_bytes())),
682        Unit::Utf16 { big: true } => text
683            .encode_utf16()
684            .for_each(|u| out.extend_from_slice(&u.to_be_bytes())),
685        Unit::Utf32 { big: false } => text
686            .chars()
687            .for_each(|c| out.extend_from_slice(&u32::from(c).to_le_bytes())),
688        Unit::Utf32 { big: true } => text
689            .chars()
690            .for_each(|c| out.extend_from_slice(&u32::from(c).to_be_bytes())),
691    }
692}
693
694struct Decoder {
695    text: String,
696    unit: Unit,
697    invalid: InvalidRuns,
698    diagnostics: Vec<Diagnostic>,
699    /// How many runs get a built diagnostic.
700    keep: usize,
701    /// How many runs to record before decoding stops ([`first_error`]).
702    max_runs: usize,
703    /// A run past `max_runs` began: decoding stopped.
704    full: bool,
705}
706
707impl Decoder {
708    /// Records the invalid bytes `input_start..input_end`, merging them into
709    /// the previous run when adjacent (into the shared pool).
710    fn invalid(&mut self, input: &[u8], input_start: usize, input_end: usize) {
711        let bytes = input.get(input_start..input_end).unwrap_or(&[]);
712        if let Some((input_end_last, text_end_last)) = self.invalid.last_ends() {
713            if input_end_last == input_start && text_end_last == self.text.len() {
714                self.invalid.pool.extend_from_slice(bytes);
715                return;
716            }
717        }
718        if self.invalid.len() >= self.max_runs {
719            self.full = true;
720            return;
721        }
722        let at = self.text.len();
723        self.invalid.push(at, input_start, bytes);
724        self.text.push(char::REPLACEMENT_CHARACTER);
725    }
726
727    /// Turns the first `keep` runs into diagnostics (after merging;
728    /// the rest are counted, never built).
729    fn finish(&mut self) {
730        for run in self.invalid.iter().take(self.keep) {
731            self.diagnostics.push(diagnostic_of(self.unit, &run));
732        }
733        // The records keep exactly what they hold.
734        self.invalid.shrink_to_fit();
735    }
736
737    fn utf8(&mut self, bytes: &[u8]) {
738        self.text.reserve(bytes.len());
739        let mut at = 0usize;
740        while let Some(rest) = bytes.get(at..) {
741            if rest.is_empty() {
742                break;
743            }
744            match core::str::from_utf8(rest) {
745                Ok(valid) => {
746                    self.text.push_str(valid);
747                    break;
748                }
749                Err(e) => {
750                    let good = e.valid_up_to();
751                    let valid = rest
752                        .get(..good)
753                        .and_then(|v| core::str::from_utf8(v).ok())
754                        .unwrap_or("");
755                    self.text.push_str(valid);
756                    let bad_len = e.error_len().unwrap_or(rest.len().saturating_sub(good));
757                    let start = at.saturating_add(good);
758                    // A zero-length error cannot happen; guard progress anyway.
759                    let end = start.saturating_add(bad_len.max(1)).min(bytes.len());
760                    self.invalid(bytes, start, end);
761                    if self.full {
762                        break;
763                    }
764                    at = end;
765                }
766            }
767        }
768        self.finish();
769    }
770
771    fn utf16(&mut self, bytes: &[u8], big: bool) {
772        self.text.reserve(bytes.len());
773        let units = bytes.chunks_exact(2).map(|p| match p {
774            [a, b] if big => u16::from_be_bytes([*a, *b]),
775            [a, b] => u16::from_le_bytes([*a, *b]),
776            _ => 0,
777        });
778        let mut at = 0usize;
779        for r in char::decode_utf16(units) {
780            match r {
781                Ok(c) => {
782                    self.text.push(c);
783                    at = at.saturating_add(c.len_utf16().saturating_mul(2));
784                }
785                Err(_) => {
786                    let end = at.saturating_add(2);
787                    self.invalid(bytes, at, end);
788                    if self.full {
789                        break;
790                    }
791                    at = end;
792                }
793            }
794        }
795        if at < bytes.len() {
796            self.invalid(bytes, at, bytes.len());
797        }
798        self.finish();
799    }
800
801    fn utf32(&mut self, bytes: &[u8], big: bool) {
802        self.text.reserve(bytes.len() / 4);
803        let mut at = 0usize;
804        for unit in bytes.chunks_exact(4) {
805            let v = match unit {
806                [a, b, c, d] if big => u32::from_be_bytes([*a, *b, *c, *d]),
807                [a, b, c, d] => u32::from_le_bytes([*a, *b, *c, *d]),
808                _ => u32::MAX,
809            };
810            let end = at.saturating_add(4);
811            match char::from_u32(v) {
812                Some(c) => self.text.push(c),
813                None => self.invalid(bytes, at, end),
814            }
815            if self.full {
816                break;
817            }
818            at = end;
819        }
820        if at < bytes.len() {
821            self.invalid(bytes, at, bytes.len());
822        }
823        self.finish();
824    }
825}
826
827#[cfg(test)]
828mod tests {
829    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
830    use super::*;
831
832    #[test]
833    fn detects_the_spec_table() {
834        // Moved from `oneyaml-core` with `Encoding::detect`.
835        let cases: [(&[u8], Encoding); 10] = [
836            (&[0, 0, 0xFE, 0xFF], Encoding::Utf32Be),
837            (&[0, 0, 0, b'a'], Encoding::Utf32Be),
838            (&[0xFF, 0xFE, 0, 0], Encoding::Utf32Le),
839            (&[b'a', 0, 0, 0], Encoding::Utf32Le),
840            (&[0xFE, 0xFF], Encoding::Utf16Be),
841            (&[0, b'a'], Encoding::Utf16Be),
842            (&[0xFF, 0xFE], Encoding::Utf16Le),
843            (&[b'a', 0], Encoding::Utf16Le),
844            (b"a: 1", Encoding::Utf8),
845            (&[], Encoding::Utf8),
846        ];
847        for (bytes, enc) in cases {
848            assert_eq!(detect_encoding(bytes), enc);
849            assert_eq!(decode(bytes).encoding(), enc);
850            assert_eq!(decode(bytes).encoder().encoding(), enc);
851        }
852    }
853
854    #[test]
855    fn every_encoding_has_a_codec() {
856        for enc in [
857            Encoding::Utf8,
858            Encoding::Utf16Le,
859            Encoding::Utf16Be,
860            Encoding::Utf32Le,
861            Encoding::Utf32Be,
862        ] {
863            let e = Encoder::new(enc).unwrap();
864            assert_eq!(e.encoding(), enc);
865            let bytes = encode("a: é\n", enc).unwrap();
866            assert_eq!(bytes, e.encode("a: é\n"));
867            let d = decode(&bytes);
868            assert_eq!((d.encoding(), d.text()), (enc, "a: é\n"));
869        }
870        assert_eq!(Encoder::default().encoding(), Encoding::Utf8);
871    }
872
873    /// Narrow records that meet an offset past `u32` become wide
874    /// (no run dropped), and equality and hashing see the runs, never the
875    /// storage form.
876    #[test]
877    fn records_widen_and_compare_by_runs() {
878        use std::hash::BuildHasher;
879        let mut narrow = InvalidRuns::for_input(10);
880        let mut wide = InvalidRuns::for_input(usize::MAX);
881        assert!(matches!(narrow.recs, Recs::Narrow(_)));
882        assert!(matches!(wide.recs, Recs::Wide(_)));
883        for r in [&mut narrow, &mut wide] {
884            r.push(0, 0, b"\x80");
885            r.push(5, 3, b"\xff\xfe");
886            r.shrink_to_fit();
887        }
888        assert_eq!(narrow, wide);
889        let h = std::hash::RandomState::new();
890        assert_eq!(h.hash_one(&narrow), h.hash_one(&wide));
891        assert_eq!(format!("{narrow:?}"), format!("{wide:?}"));
892        let big = usize::try_from(u64::from(u32::MAX) + 7).unwrap();
893        narrow.push(big, big, b"\xc3");
894        assert!(matches!(narrow.recs, Recs::Wide(_)));
895        assert_eq!(narrow.len(), 3);
896        let runs: Vec<_> = narrow
897            .iter()
898            .map(|r| (r.text.start, r.input, r.bytes))
899            .collect();
900        assert_eq!(
901            runs,
902            [
903                (0, Span::new(0, 1), b"\x80".as_slice()),
904                (5, Span::new(3, 5), b"\xff\xfe".as_slice()),
905                (big, Span::new(big, big + 1), b"\xc3".as_slice()),
906            ]
907        );
908        assert_ne!(narrow, wide);
909    }
910}