#![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
use oneyaml_core::Span;
use oneyaml_core::code::{INVALID_UTF8, INVALID_UTF16, INVALID_UTF32};
use oneyaml_syntax::event::{EventKind, ScalarStyle};
use oneyaml_syntax::{Encoding, check_contract, decode, decode_strict, encode, first_error, parse};
const ALL: [Encoding; 5] = [
Encoding::Utf8,
Encoding::Utf16Le,
Encoding::Utf16Be,
Encoding::Utf32Le,
Encoding::Utf32Be,
];
const SAMPLE: &str = "a: é\r\nb: [\"\u{1F600}\", 'ü']\n# ✓\n...\n\u{feff}--- |\n x\u{10FFFF}\n";
fn parse_round_trip(text: &str) -> Vec<String> {
let events = parse(text);
check_contract(text, &events).unwrap();
let rebuilt: String = events.sourced().map(|e| e.text().unwrap()).collect();
assert_eq!(rebuilt.as_bytes(), text.as_bytes());
events
.sourced()
.filter_map(|e| match e.kind() {
EventKind::Scalar { .. } => e.value().map(|v| v.to_string()),
_ => None,
})
.collect()
}
#[test]
fn every_encoding_round_trips_byte_for_byte() {
let expected = parse_round_trip(SAMPLE);
assert_eq!(expected, ["a", "é", "b", "\u{1F600}", "ü", "x\u{10FFFF}\n"]);
for enc in ALL {
for bom in [false, true] {
let text = if bom {
format!("\u{feff}{SAMPLE}")
} else {
SAMPLE.to_owned()
};
let bytes = encode(&text, enc).unwrap();
let d = decode(&bytes);
assert_eq!(d.encoding(), enc, "{enc:?} bom={bom}");
assert!(d.is_valid() && d.diagnostics().is_empty());
assert_eq!(d.has_bom(), bom);
assert_eq!(d.text(), text);
assert_eq!(parse_round_trip(d.text()), expected);
assert_eq!(encode(d.text(), d.encoding()).unwrap(), bytes);
assert_eq!(d.reencode(), bytes);
}
}
}
#[test]
fn valid_utf8_is_borrowed() {
let bytes = b"\xEF\xBB\xBFa: b\n";
let d = decode(bytes);
assert!(matches!(
d.clone().into_text(),
std::borrow::Cow::Borrowed(_)
));
assert_eq!(d.encoding(), Encoding::Utf8);
assert!(d.has_bom());
assert_eq!(d.text().as_bytes(), bytes);
}
#[test]
fn detection_follows_the_spec_table() {
let cases: &[(&[u8], Encoding)] = &[
(b"", Encoding::Utf8),
(b"a", Encoding::Utf8),
(b"\xEF\xBB\xBFa", Encoding::Utf8),
(b"\x00\x00\xFE\xFF", Encoding::Utf32Be),
(b"\x00\x00\x00a", Encoding::Utf32Be),
(b"\xFF\xFE\x00\x00", Encoding::Utf32Le),
(b"a\x00\x00\x00", Encoding::Utf32Le),
(b"\xFE\xFF", Encoding::Utf16Be),
(b"\x00a", Encoding::Utf16Be),
(b"\xFF\xFE", Encoding::Utf16Le),
(b"a\x00", Encoding::Utf16Le),
(b"\x00\x00\x00", Encoding::Utf16Be),
(b"\xE9\x00", Encoding::Utf16Le),
(b"\xC3\xA9", Encoding::Utf8),
];
for (bytes, enc) in cases {
assert_eq!(oneyaml_syntax::detect_encoding(bytes), *enc, "{bytes:?}");
}
let bytes = encode("\u{e9e9}: x", Encoding::Utf16Be).unwrap();
let d = decode(&bytes);
assert_eq!(d.encoding(), Encoding::Utf8);
assert!(!d.is_valid());
}
fn check_invalid(bytes: &[u8], enc: Encoding, text: &str, runs: &[(usize, usize)]) {
let d = decode(bytes);
assert_eq!(d.encoding(), enc, "{bytes:?}");
assert_eq!(d.text(), text, "{bytes:?}");
assert!(!d.is_valid());
let got: Vec<(usize, usize)> = d
.invalid_runs()
.iter()
.map(|r| (r.input.start, r.input.end))
.collect();
assert_eq!(got, runs, "{bytes:?}");
let code = match enc {
Encoding::Utf8 => INVALID_UTF8,
Encoding::Utf16Le | Encoding::Utf16Be => INVALID_UTF16,
_ => INVALID_UTF32,
};
assert_eq!(d.diagnostics().len(), runs.len());
for (diag, run) in d.diagnostics().iter().zip(d.invalid_runs()) {
assert_eq!(diag.code, code);
assert_eq!(diag.severity, oneyaml_core::Severity::Error);
assert_eq!(diag.primary.span, run.text);
assert_eq!(d.text().get(run.text.start..run.text.end), Some("\u{FFFD}"));
assert_eq!(&bytes[run.input.start..run.input.end], run.bytes);
assert!(diag.help.as_deref().unwrap().contains("§5.2"));
}
parse_round_trip(d.text());
assert_eq!(d.reencode(), bytes, "byte-identical for {bytes:?}");
}
#[test]
fn malformed_utf8() {
check_invalid(b"a: \x80b\n", Encoding::Utf8, "a: \u{FFFD}b\n", &[(3, 4)]);
check_invalid(b"a\xE2\x82", Encoding::Utf8, "a\u{FFFD}", &[(1, 3)]);
check_invalid(b"a\xC0\xAFb", Encoding::Utf8, "a\u{FFFD}b", &[(1, 3)]);
check_invalid(b"\xED\xA0\x80", Encoding::Utf8, "\u{FFFD}", &[(0, 3)]);
check_invalid(
b"\xFFa\xFE\xFF",
Encoding::Utf8,
"\u{FFFD}a\u{FFFD}",
&[(0, 1), (2, 4)],
);
check_invalid(
b"x\xFFy\xFF\xFE",
Encoding::Utf8,
"x\u{FFFD}y\u{FFFD}",
&[(1, 2), (3, 5)],
);
}
#[test]
fn malformed_utf16() {
let hi = [b'a', 0, 0x00, 0xD8, b'b', 0];
check_invalid(&hi, Encoding::Utf16Le, "a\u{FFFD}b", &[(2, 4)]);
let lo = [0, b'a', 0xDC, 0x00];
check_invalid(&lo, Encoding::Utf16Be, "a\u{FFFD}", &[(2, 4)]);
let reversed = [0, b'a', 0xDC, 0x00, 0xD8, 0x00, 0, b'b'];
check_invalid(&reversed, Encoding::Utf16Be, "a\u{FFFD}b", &[(2, 6)]);
let odd = [b'a', 0, b'b'];
check_invalid(&odd, Encoding::Utf16Le, "a\u{FFFD}", &[(2, 3)]);
let bom_odd = [0xFE, 0xFF, 0];
check_invalid(&bom_odd, Encoding::Utf16Be, "\u{feff}\u{FFFD}", &[(2, 3)]);
}
#[test]
fn malformed_utf32() {
let surrogate = [b'a', 0, 0, 0, 0x00, 0xD8, 0, 0];
check_invalid(&surrogate, Encoding::Utf32Le, "a\u{FFFD}", &[(4, 8)]);
let big = [0, 0, 0, b'a', 0x00, 0x11, 0x00, 0x00, 0, 0, 0, b'b'];
check_invalid(&big, Encoding::Utf32Be, "a\u{FFFD}b", &[(4, 8)]);
let tail = [0, 0, 0, b'a', 0, 0];
check_invalid(&tail, Encoding::Utf32Be, "a\u{FFFD}", &[(4, 6)]);
let both = [b'a', 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 1];
check_invalid(&both, Encoding::Utf32Le, "a\u{FFFD}", &[(4, 9)]);
}
#[test]
fn garbage_never_panics_and_always_reencodes() {
let mut x: u64 = 0x9e37_79b9_7f4a_7c15;
let pool = [
0u8, 0, 0xD8, 0xDC, 0xFE, 0xFF, 0xEF, 0xBB, 0xBF, b'a', b':', b' ', b'\n', 0x80, 0xC3,
];
for _ in 0..5_000 {
x = x.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
let len = usize::try_from(x.wrapping_shr(59)).unwrap();
let bytes: Vec<u8> = (0..len)
.map(|k| {
x = x.rotate_left(7) ^ u64::try_from(k).unwrap();
pool[usize::try_from(x.checked_rem(15).unwrap()).unwrap()]
})
.collect();
let d = decode(&bytes);
assert_eq!(d.reencode(), bytes);
assert_eq!(d.diagnostics().len(), d.invalid_runs().len());
match decode_strict(&bytes) {
Ok(s) => {
assert!(d.is_valid());
assert_eq!((s.text(), s.encoding()), (d.text(), d.encoding()));
}
Err(e) => {
assert_eq!(Some(*e.clone()), first_error(&bytes));
assert_eq!(e.0, d.diagnostics()[0]);
}
}
let events = parse(d.text());
check_contract(d.text(), &events).unwrap();
}
}
#[test]
fn decoded_text_feeds_the_parser() {
let bytes = encode("\u{feff}k: 'v'\n", Encoding::Utf16Be).unwrap();
let d = decode(&bytes);
let events = parse(d.text());
let scalar = events
.sourced()
.find_map(|e| match e.kind() {
EventKind::Scalar {
style: ScalarStyle::SingleQuoted,
..
} => Some((e.value().unwrap(), e.span())),
_ => None,
})
.unwrap();
assert_eq!(scalar.0, "v");
assert_eq!(scalar.1, Span::new(6, 9));
}