use super::text::scan_str;
use super::{Hit, Unreadable};
pub(crate) fn looks_binary(bytes: &[u8]) -> bool {
bytes.contains(&0)
}
pub type Scanned<'a> = (&'a str, Vec<Hit>);
pub fn scan_bytes(
bytes: &[u8],
allowed: impl Fn(char) -> bool,
) -> Result<Scanned<'_>, Unreadable> {
decode(bytes).map(|text| (text, scan_str(text, allowed)))
}
pub(crate) fn decode(bytes: &[u8]) -> Result<&str, Unreadable> {
if looks_binary(bytes) {
return Err(Unreadable::Binary);
}
core::str::from_utf8(bytes)
.map_err(|error| not_utf8(bytes, error.valid_up_to()))
}
pub(crate) fn decode_owned(bytes: Vec<u8>) -> Result<String, Unreadable> {
if looks_binary(&bytes) {
return Err(Unreadable::Binary);
}
String::from_utf8(bytes).map_err(|error| {
let byte = error.utf8_error().valid_up_to();
not_utf8(error.as_bytes(), byte)
})
}
fn not_utf8(bytes: &[u8], byte: usize) -> Unreadable {
let before = bytes.get(..byte).and_then(|b| core::str::from_utf8(b).ok());
let before = before.unwrap_or_default();
let line = before.matches('\n').count().saturating_add(1);
let tail = before.rsplit('\n').next().unwrap_or_default();
let column = tail.chars().count().saturating_add(1);
Unreadable::NotUtf8 { byte, line, column }
}
#[cfg(test)]
mod tests {
use super::{decode, decode_owned, looks_binary, scan_bytes};
use crate::scan::{Hit, Position, Unreadable};
fn ascii_only(character: char) -> bool {
character.is_ascii()
}
#[test]
fn invalid_utf8_reports_the_byte_offset_where_decoding_failed() {
let outcome = scan_bytes(b"ab\xffcd", ascii_only);
assert_eq!(
outcome,
Err(Unreadable::NotUtf8 {
byte: 2,
line: 1,
column: 3
})
);
}
#[test]
fn invalid_utf8_is_located_by_line_and_column() {
let outcome = decode(b"ok\n\xc3\xa9x\xff");
let want = Unreadable::NotUtf8 {
byte: 6,
line: 2,
column: 3,
};
assert_eq!(outcome, Err(want));
}
#[test]
fn a_nul_byte_makes_the_input_binary() {
let outcome = scan_bytes(b"ab\x00cd", ascii_only);
assert_eq!(outcome, Err(Unreadable::Binary));
}
#[test]
fn binary_is_decided_before_utf8_so_a_skip_is_not_an_error() {
let outcome = scan_bytes(b"\x00\xff", ascii_only);
assert_eq!(outcome, Err(Unreadable::Binary));
}
#[test]
fn readable_bytes_are_scanned_as_text() {
let text = "ab\u{200b}";
let outcome = scan_bytes(text.as_bytes(), ascii_only);
let (line, column, byte) = (1, 3, 2);
let position = Position { line, column, byte };
let character = '\u{200b}';
let hit = Hit {
position,
character,
};
assert_eq!(outcome, Ok((text, vec![hit])));
}
#[test]
fn the_owned_decode_agrees_with_the_borrowed_one() {
for bytes in [&b"ab\xffcd"[..], b"ab\x00cd", b"\x00\xff", b"ok\n"] {
let owned = decode_owned(bytes.to_vec());
let borrowed = decode(bytes).map(str::to_owned);
assert_eq!(owned, borrowed);
}
}
#[test]
fn decoding_refuses_the_way_scanning_does() {
assert_eq!(decode(b"ab\x00\xff"), Err(Unreadable::Binary));
assert_eq!(
decode(b"ab\xff"),
Err(Unreadable::NotUtf8 {
byte: 2,
line: 1,
column: 3
})
);
assert_eq!(decode("a\u{e9}".as_bytes()), Ok("a\u{e9}"));
}
#[test]
fn the_heuristic_stands_alone_and_looks_only_for_a_nul() {
assert!(looks_binary(b"\x00"));
assert!(!looks_binary(b"\xff\xfe plain high bytes"));
}
}