use std::borrow::Cow;
use oneyaml_core::code::{INTERNAL_ERROR, INVALID_UTF8, INVALID_UTF16, INVALID_UTF32};
use oneyaml_core::{Code, Diagnostic, Label, Span};
pub use oneyaml_core::Encoding;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum Unit {
Utf8,
Utf16 { big: bool },
Utf32 { big: bool },
}
const fn encoding_of(unit: Unit) -> Encoding {
match unit {
Unit::Utf8 => Encoding::Utf8,
Unit::Utf16 { big: false } => Encoding::Utf16Le,
Unit::Utf16 { big: true } => Encoding::Utf16Be,
Unit::Utf32 { big: false } => Encoding::Utf32Le,
Unit::Utf32 { big: true } => Encoding::Utf32Be,
}
}
fn unit(encoding: Encoding) -> Result<Unit, Box<Diagnostic>> {
match encoding {
Encoding::Utf8 => Ok(Unit::Utf8),
Encoding::Utf16Le => Ok(Unit::Utf16 { big: false }),
Encoding::Utf16Be => Ok(Unit::Utf16 { big: true }),
Encoding::Utf32Le => Ok(Unit::Utf32 { big: false }),
Encoding::Utf32Be => Ok(Unit::Utf32 { big: true }),
_ => {
debug_assert!(false, "no codec for {encoding:?}");
Err(Box::new(Diagnostic::error(
INTERNAL_ERROR,
format!("no codec for the encoding {}", encoding.name()),
Label::new(Span::empty(0), ""),
)))
}
}
}
const fn detect_unit(bytes: &[u8]) -> Unit {
match bytes {
[0, 0, 0xFE, 0xFF, ..] | [0, 0, 0, _, ..] => Unit::Utf32 { big: true },
[0xFF, 0xFE, 0, 0, ..] | [_, 0, 0, 0, ..] => Unit::Utf32 { big: false },
[0xFE, 0xFF, ..] | [0, _, ..] => Unit::Utf16 { big: true },
[0xFF, 0xFE, ..] | [_, 0, ..] => Unit::Utf16 { big: false },
_ => Unit::Utf8,
}
}
#[must_use]
pub const fn detect_encoding(bytes: &[u8]) -> Encoding {
encoding_of(detect_unit(bytes))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Encoder(Unit);
impl Default for Encoder {
fn default() -> Self {
Self(Unit::Utf8)
}
}
impl Encoder {
pub fn new(encoding: Encoding) -> Result<Self, Box<Diagnostic>> {
unit(encoding).map(Self)
}
#[must_use]
pub const fn encoding(self) -> Encoding {
encoding_of(self.0)
}
#[must_use]
pub fn encode(self, text: &str) -> Vec<u8> {
let mut out = Vec::new();
encode_into(&mut out, text, self.0);
out
}
}
const fn error_code(unit: Unit) -> Code {
match unit {
Unit::Utf8 => INVALID_UTF8,
Unit::Utf16 { .. } => INVALID_UTF16,
Unit::Utf32 { .. } => INVALID_UTF32,
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct InvalidRun<'r> {
pub text: Span,
pub input: Span,
pub bytes: &'r [u8],
}
const REPLACEMENT_LEN: usize = char::REPLACEMENT_CHARACTER.len_utf8();
#[derive(Clone, Default)]
pub struct InvalidRuns {
recs: Recs,
pool: Vec<u8>,
}
impl PartialEq for InvalidRuns {
fn eq(&self, other: &Self) -> bool {
self.len() == other.len() && self.iter().eq(other.iter())
}
}
impl Eq for InvalidRuns {}
impl core::hash::Hash for InvalidRuns {
fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
state.write_usize(self.len());
for run in self {
run.hash(state);
}
}
}
impl core::fmt::Debug for InvalidRuns {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_list().entries(self.iter()).finish()
}
}
#[derive(Clone, Debug)]
enum Recs {
Narrow(Vec<[u32; 3]>),
Wide(Vec<[usize; 3]>),
}
impl Default for Recs {
fn default() -> Self {
Self::Narrow(Vec::new())
}
}
impl Recs {
fn len(&self) -> usize {
match self {
Self::Narrow(v) => v.len(),
Self::Wide(v) => v.len(),
}
}
fn get(&self, i: usize) -> Option<[usize; 3]> {
match self {
Self::Narrow(v) => v.get(i).map(|r| r.map(|x| x as usize)),
Self::Wide(v) => v.get(i).copied(),
}
}
fn push(&mut self, rec: [usize; 3]) {
if let Self::Narrow(v) = self {
if let [Ok(a), Ok(b), Ok(c)] = rec.map(u32::try_from) {
v.push([a, b, c]);
return;
}
*self = Self::Wide(v.iter().map(|r| r.map(|x| x as usize)).collect());
}
if let Self::Wide(v) = self {
v.push(rec);
}
}
fn shrink_to_fit(&mut self) {
match self {
Self::Narrow(v) => v.shrink_to_fit(),
Self::Wide(v) => v.shrink_to_fit(),
}
}
fn first_at(&self, at: usize) -> usize {
match self {
Self::Narrow(v) => v.partition_point(|r| (r[0] as usize) < at),
Self::Wide(v) => v.partition_point(|r| r[0] < at),
}
}
fn heap_bytes(&self) -> usize {
match self {
Self::Narrow(v) => v.capacity().saturating_mul(size_of::<[u32; 3]>()),
Self::Wide(v) => v.capacity().saturating_mul(size_of::<[usize; 3]>()),
}
}
}
impl InvalidRuns {
fn for_input(len: usize) -> Self {
let narrow = len
.checked_mul(REPLACEMENT_LEN)
.is_some_and(|x| u32::try_from(x).is_ok());
let recs = if narrow {
Recs::Narrow(Vec::new())
} else {
Recs::Wide(Vec::new())
};
Self {
recs,
pool: Vec::new(),
}
}
#[must_use]
pub fn len(&self) -> usize {
self.recs.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
#[must_use]
pub fn get(&self, i: usize) -> Option<InvalidRun<'_>> {
let [text, input, from] = self.recs.get(i)?;
let to = match self.recs.get(i.checked_add(1)?) {
Some([_, _, next]) => next,
None => self.pool.len(),
};
let bytes = self.pool.get(from..to)?;
Some(InvalidRun {
text: Span::new(text, text.saturating_add(REPLACEMENT_LEN)),
input: Span::new(input, input.saturating_add(bytes.len())),
bytes,
})
}
#[must_use]
pub fn first_at(&self, at: usize) -> usize {
self.recs.first_at(at)
}
#[must_use]
pub fn iter(&self) -> InvalidRunsIter<'_> {
InvalidRunsIter {
runs: self,
at: 0,
end: self.len(),
}
}
#[must_use]
#[doc(hidden)]
pub fn heap_bytes(&self) -> usize {
self.recs.heap_bytes().saturating_add(self.pool.capacity())
}
fn push(&mut self, text: usize, input: usize, bytes: &[u8]) {
self.recs.push([text, input, self.pool.len()]);
self.pool.extend_from_slice(bytes);
}
fn last_ends(&self) -> Option<(usize, usize)> {
let r = self.get(self.len().checked_sub(1)?)?;
Some((r.input.end, r.text.end))
}
fn shrink_to_fit(&mut self) {
self.recs.shrink_to_fit();
self.pool.shrink_to_fit();
}
}
impl<'r> IntoIterator for &'r InvalidRuns {
type Item = InvalidRun<'r>;
type IntoIter = InvalidRunsIter<'r>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
#[derive(Clone, Debug)]
pub struct InvalidRunsIter<'r> {
runs: &'r InvalidRuns,
at: usize,
end: usize,
}
impl<'r> Iterator for InvalidRunsIter<'r> {
type Item = InvalidRun<'r>;
fn next(&mut self) -> Option<Self::Item> {
if self.at >= self.end {
return None;
}
let r = self.runs.get(self.at);
self.at = self.at.saturating_add(1);
r
}
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.at = self.at.saturating_add(n).min(self.end);
self.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
let n = self.end.saturating_sub(self.at);
(n, Some(n))
}
}
impl DoubleEndedIterator for InvalidRunsIter<'_> {
fn next_back(&mut self) -> Option<Self::Item> {
if self.at >= self.end {
return None;
}
self.end = self.end.saturating_sub(1);
self.runs.get(self.end)
}
}
impl ExactSizeIterator for InvalidRunsIter<'_> {}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Decoded<'a> {
text: Cow<'a, str>,
unit: Unit,
invalid: InvalidRuns,
diagnostics: Vec<Diagnostic>,
suppressed: usize,
}
impl<'a> Decoded<'a> {
#[must_use]
pub fn text(&self) -> &str {
&self.text
}
#[must_use]
pub fn into_text(self) -> Cow<'a, str> {
self.text
}
#[must_use]
pub const fn encoding(&self) -> Encoding {
encoding_of(self.unit)
}
#[must_use]
pub const fn encoder(&self) -> Encoder {
Encoder(self.unit)
}
#[must_use]
#[doc(hidden)]
pub fn has_bom(&self) -> bool {
self.text.starts_with('\u{feff}')
}
#[must_use]
pub fn invalid_runs(&self) -> &InvalidRuns {
&self.invalid
}
#[must_use]
pub fn diagnostics(&self) -> &[Diagnostic] {
&self.diagnostics
}
#[must_use]
pub const fn suppressed_diagnostics(&self) -> usize {
self.suppressed
}
#[must_use]
pub fn is_valid(&self) -> bool {
self.invalid.is_empty()
}
#[must_use]
pub fn into_parts(self) -> (Vec<Diagnostic>, InvalidRuns, Encoder, Cow<'a, str>) {
(
self.diagnostics,
self.invalid,
Encoder(self.unit),
self.text,
)
}
#[must_use]
pub fn reencode(&self) -> Vec<u8> {
let mut out = Vec::with_capacity(self.text.len());
let mut at = 0usize;
for run in &self.invalid {
let before = self.text.get(at..run.text.start).unwrap_or("");
encode_into(&mut out, before, self.unit);
out.extend_from_slice(run.bytes);
at = run.text.end;
}
encode_into(&mut out, self.text.get(at..).unwrap_or(""), self.unit);
out
}
}
#[must_use]
pub fn decode(bytes: &[u8]) -> Decoded<'_> {
decode_with_limit(bytes, usize::MAX)
}
#[must_use]
pub fn decode_with_limit(bytes: &[u8], keep: usize) -> Decoded<'_> {
let unit = detect_unit(bytes);
let mut d = Decoder {
text: String::new(),
unit,
invalid: InvalidRuns::for_input(bytes.len()),
diagnostics: Vec::new(),
keep,
max_runs: usize::MAX,
full: false,
};
match unit {
Unit::Utf8 => {
if let Ok(text) = core::str::from_utf8(bytes) {
return Decoded {
text: Cow::Borrowed(text),
unit,
invalid: InvalidRuns::default(),
diagnostics: Vec::new(),
suppressed: 0,
};
}
d.utf8(bytes);
}
Unit::Utf16 { big } => d.utf16(bytes, big),
Unit::Utf32 { big } => d.utf32(bytes, big),
}
let suppressed = d.invalid.len().saturating_sub(d.diagnostics.len());
Decoded {
text: Cow::Owned(d.text),
unit,
invalid: d.invalid,
diagnostics: d.diagnostics,
suppressed,
}
}
#[must_use]
pub fn first_error(bytes: &[u8]) -> Option<(Diagnostic, String)> {
let unit = detect_unit(bytes);
if matches!(unit, Unit::Utf8) && core::str::from_utf8(bytes).is_ok() {
return None;
}
let mut d = Decoder {
text: String::new(),
unit,
invalid: InvalidRuns::for_input(bytes.len()),
diagnostics: Vec::new(),
keep: 1,
max_runs: 1,
full: false,
};
match unit {
Unit::Utf8 => d.utf8(bytes),
Unit::Utf16 { big } => d.utf16(bytes, big),
Unit::Utf32 { big } => d.utf32(bytes, big),
}
let text = d.text;
d.diagnostics.into_iter().next().map(|e| (e, text))
}
pub fn decode_strict(bytes: &[u8]) -> Result<Decoded<'_>, Box<(Diagnostic, String)>> {
let unit = detect_unit(bytes);
if matches!(unit, Unit::Utf8) {
if let Ok(text) = core::str::from_utf8(bytes) {
return Ok(Decoded {
text: Cow::Borrowed(text),
unit,
invalid: InvalidRuns::default(),
diagnostics: Vec::new(),
suppressed: 0,
});
}
}
let mut d = Decoder {
text: String::new(),
unit,
invalid: InvalidRuns::for_input(bytes.len()),
diagnostics: Vec::new(),
keep: 1,
max_runs: 1,
full: false,
};
match unit {
Unit::Utf8 => d.utf8(bytes),
Unit::Utf16 { big } => d.utf16(bytes, big),
Unit::Utf32 { big } => d.utf32(bytes, big),
}
match d.diagnostics.into_iter().next() {
Some(e) => Err(Box::new((e, d.text))),
None => Ok(Decoded {
text: Cow::Owned(d.text),
unit,
invalid: d.invalid,
diagnostics: Vec::new(),
suppressed: 0,
}),
}
}
#[must_use]
pub fn run_diagnostic(encoding: Encoding, run: &InvalidRun<'_>) -> Diagnostic {
let unit = match encoding {
Encoding::Utf16Le => Unit::Utf16 { big: false },
Encoding::Utf16Be => Unit::Utf16 { big: true },
Encoding::Utf32Le => Unit::Utf32 { big: false },
Encoding::Utf32Be => Unit::Utf32 { big: true },
_ => Unit::Utf8,
};
diagnostic_of(unit, run)
}
fn diagnostic_of(unit: Unit, run: &InvalidRun<'_>) -> Diagnostic {
let name = encoding_of(unit).name();
let message = format!(
"invalid {name} at input bytes {}..{}",
run.input.start, run.input.end
);
Diagnostic::error(
error_code(unit),
message,
Label::new(run.text, "replaced with U+FFFD"),
)
.with_help(format!(
"the input was read as {name} (YAML 1.2.2 §5.2: detected from its first bytes); \
save it as valid {name} or UTF-8"
))
}
pub fn encode(text: &str, encoding: Encoding) -> Result<Vec<u8>, Box<Diagnostic>> {
Encoder::new(encoding).map(|e| e.encode(text))
}
fn encode_into(out: &mut Vec<u8>, text: &str, unit: Unit) {
match unit {
Unit::Utf8 => out.extend_from_slice(text.as_bytes()),
Unit::Utf16 { big: false } => text
.encode_utf16()
.for_each(|u| out.extend_from_slice(&u.to_le_bytes())),
Unit::Utf16 { big: true } => text
.encode_utf16()
.for_each(|u| out.extend_from_slice(&u.to_be_bytes())),
Unit::Utf32 { big: false } => text
.chars()
.for_each(|c| out.extend_from_slice(&u32::from(c).to_le_bytes())),
Unit::Utf32 { big: true } => text
.chars()
.for_each(|c| out.extend_from_slice(&u32::from(c).to_be_bytes())),
}
}
struct Decoder {
text: String,
unit: Unit,
invalid: InvalidRuns,
diagnostics: Vec<Diagnostic>,
keep: usize,
max_runs: usize,
full: bool,
}
impl Decoder {
fn invalid(&mut self, input: &[u8], input_start: usize, input_end: usize) {
let bytes = input.get(input_start..input_end).unwrap_or(&[]);
if let Some((input_end_last, text_end_last)) = self.invalid.last_ends() {
if input_end_last == input_start && text_end_last == self.text.len() {
self.invalid.pool.extend_from_slice(bytes);
return;
}
}
if self.invalid.len() >= self.max_runs {
self.full = true;
return;
}
let at = self.text.len();
self.invalid.push(at, input_start, bytes);
self.text.push(char::REPLACEMENT_CHARACTER);
}
fn finish(&mut self) {
for run in self.invalid.iter().take(self.keep) {
self.diagnostics.push(diagnostic_of(self.unit, &run));
}
self.invalid.shrink_to_fit();
}
fn utf8(&mut self, bytes: &[u8]) {
self.text.reserve(bytes.len());
let mut at = 0usize;
while let Some(rest) = bytes.get(at..) {
if rest.is_empty() {
break;
}
match core::str::from_utf8(rest) {
Ok(valid) => {
self.text.push_str(valid);
break;
}
Err(e) => {
let good = e.valid_up_to();
let valid = rest
.get(..good)
.and_then(|v| core::str::from_utf8(v).ok())
.unwrap_or("");
self.text.push_str(valid);
let bad_len = e.error_len().unwrap_or(rest.len().saturating_sub(good));
let start = at.saturating_add(good);
let end = start.saturating_add(bad_len.max(1)).min(bytes.len());
self.invalid(bytes, start, end);
if self.full {
break;
}
at = end;
}
}
}
self.finish();
}
fn utf16(&mut self, bytes: &[u8], big: bool) {
self.text.reserve(bytes.len());
let units = bytes.chunks_exact(2).map(|p| match p {
[a, b] if big => u16::from_be_bytes([*a, *b]),
[a, b] => u16::from_le_bytes([*a, *b]),
_ => 0,
});
let mut at = 0usize;
for r in char::decode_utf16(units) {
match r {
Ok(c) => {
self.text.push(c);
at = at.saturating_add(c.len_utf16().saturating_mul(2));
}
Err(_) => {
let end = at.saturating_add(2);
self.invalid(bytes, at, end);
if self.full {
break;
}
at = end;
}
}
}
if at < bytes.len() {
self.invalid(bytes, at, bytes.len());
}
self.finish();
}
fn utf32(&mut self, bytes: &[u8], big: bool) {
self.text.reserve(bytes.len() / 4);
let mut at = 0usize;
for unit in bytes.chunks_exact(4) {
let v = match unit {
[a, b, c, d] if big => u32::from_be_bytes([*a, *b, *c, *d]),
[a, b, c, d] => u32::from_le_bytes([*a, *b, *c, *d]),
_ => u32::MAX,
};
let end = at.saturating_add(4);
match char::from_u32(v) {
Some(c) => self.text.push(c),
None => self.invalid(bytes, at, end),
}
if self.full {
break;
}
at = end;
}
if at < bytes.len() {
self.invalid(bytes, at, bytes.len());
}
self.finish();
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
use super::*;
#[test]
fn detects_the_spec_table() {
let cases: [(&[u8], Encoding); 10] = [
(&[0, 0, 0xFE, 0xFF], Encoding::Utf32Be),
(&[0, 0, 0, b'a'], Encoding::Utf32Be),
(&[0xFF, 0xFE, 0, 0], Encoding::Utf32Le),
(&[b'a', 0, 0, 0], Encoding::Utf32Le),
(&[0xFE, 0xFF], Encoding::Utf16Be),
(&[0, b'a'], Encoding::Utf16Be),
(&[0xFF, 0xFE], Encoding::Utf16Le),
(&[b'a', 0], Encoding::Utf16Le),
(b"a: 1", Encoding::Utf8),
(&[], Encoding::Utf8),
];
for (bytes, enc) in cases {
assert_eq!(detect_encoding(bytes), enc);
assert_eq!(decode(bytes).encoding(), enc);
assert_eq!(decode(bytes).encoder().encoding(), enc);
}
}
#[test]
fn every_encoding_has_a_codec() {
for enc in [
Encoding::Utf8,
Encoding::Utf16Le,
Encoding::Utf16Be,
Encoding::Utf32Le,
Encoding::Utf32Be,
] {
let e = Encoder::new(enc).unwrap();
assert_eq!(e.encoding(), enc);
let bytes = encode("a: é\n", enc).unwrap();
assert_eq!(bytes, e.encode("a: é\n"));
let d = decode(&bytes);
assert_eq!((d.encoding(), d.text()), (enc, "a: é\n"));
}
assert_eq!(Encoder::default().encoding(), Encoding::Utf8);
}
#[test]
fn records_widen_and_compare_by_runs() {
use std::hash::BuildHasher;
let mut narrow = InvalidRuns::for_input(10);
let mut wide = InvalidRuns::for_input(usize::MAX);
assert!(matches!(narrow.recs, Recs::Narrow(_)));
assert!(matches!(wide.recs, Recs::Wide(_)));
for r in [&mut narrow, &mut wide] {
r.push(0, 0, b"\x80");
r.push(5, 3, b"\xff\xfe");
r.shrink_to_fit();
}
assert_eq!(narrow, wide);
let h = std::hash::RandomState::new();
assert_eq!(h.hash_one(&narrow), h.hash_one(&wide));
assert_eq!(format!("{narrow:?}"), format!("{wide:?}"));
let big = usize::try_from(u64::from(u32::MAX) + 7).unwrap();
narrow.push(big, big, b"\xc3");
assert!(matches!(narrow.recs, Recs::Wide(_)));
assert_eq!(narrow.len(), 3);
let runs: Vec<_> = narrow
.iter()
.map(|r| (r.text.start, r.input, r.bytes))
.collect();
assert_eq!(
runs,
[
(0, Span::new(0, 1), b"\x80".as_slice()),
(5, Span::new(3, 5), b"\xff\xfe".as_slice()),
(big, Span::new(big, big + 1), b"\xc3".as_slice()),
]
);
assert_ne!(narrow, wide);
}
}